diff options
Diffstat (limited to 'libstdc++-v3/include')
25 files changed, 1015 insertions, 1003 deletions
diff --git a/libstdc++-v3/include/bits/atomic_timed_wait.h b/libstdc++-v3/include/bits/atomic_timed_wait.h index 9a6ac95..bd2e6bf 100644 --- a/libstdc++-v3/include/bits/atomic_timed_wait.h +++ b/libstdc++-v3/include/bits/atomic_timed_wait.h @@ -37,7 +37,6 @@ #include <bits/atomic_wait.h> #if __glibcxx_atomic_wait -#include <bits/functional_hash.h> #include <bits/this_thread_sleep.h> #include <bits/chrono.h> @@ -70,383 +69,159 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION _Dur>& __atime) noexcept { using __w_dur = typename __wait_clock_t::duration; - return chrono::ceil<__w_dur>(__atime); + if constexpr (is_same_v<__w_dur, _Dur>) + return __atime; + else + return chrono::ceil<__w_dur>(__atime); } #ifdef _GLIBCXX_HAVE_LINUX_FUTEX #define _GLIBCXX_HAVE_PLATFORM_TIMED_WAIT - // returns true if wait ended before timeout - template<typename _Dur> - bool - __platform_wait_until_impl(const __platform_wait_t* __addr, - __platform_wait_t __old, - const chrono::time_point<__wait_clock_t, _Dur>& - __atime) noexcept - { - auto __s = chrono::time_point_cast<chrono::seconds>(__atime); - auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s); - - struct timespec __rt = - { - static_cast<std::time_t>(__s.time_since_epoch().count()), - static_cast<long>(__ns.count()) - }; - - auto __e = syscall (SYS_futex, __addr, - static_cast<int>(__futex_wait_flags:: - __wait_bitset_private), - __old, &__rt, nullptr, - static_cast<int>(__futex_wait_flags:: - __bitset_match_any)); - - if (__e) - { - if (errno == ETIMEDOUT) - return false; - if (errno != EINTR && errno != EAGAIN) - __throw_system_error(errno); - } - return true; - } - - // returns true if wait ended before timeout - template<typename _Clock, typename _Dur> - bool - __platform_wait_until(const __platform_wait_t* __addr, __platform_wait_t __old, - const chrono::time_point<_Clock, _Dur>& __atime) - { - if constexpr (is_same_v<__wait_clock_t, _Clock>) - { - return __platform_wait_until_impl(__addr, __old, __atime); - } - else - { - if (!__platform_wait_until_impl(__addr, __old, - __to_wait_clock(__atime))) - { - // We got a timeout when measured against __clock_t but - // we need to check against the caller-supplied clock - // to tell whether we should return a timeout. - if (_Clock::now() < __atime) - return true; - } - return false; - } - } #else -// define _GLIBCXX_HAVE_PLATFORM_TIMED_WAIT and implement __platform_wait_until() +// define _GLIBCXX_HAVE_PLATFORM_TIMED_WAIT and implement __platform_wait_until // if there is a more efficient primitive supported by the platform -// (e.g. __ulock_wait())which is better than pthread_cond_clockwait -#endif // ! PLATFORM_TIMED_WAIT - -#ifdef _GLIBCXX_HAS_GTHREADS - // Returns true if wait ended before timeout. - // _Clock must be either steady_clock or system_clock. - template<typename _Clock, typename _Dur> - bool - __cond_wait_until_impl(__condvar& __cv, mutex& __mx, - const chrono::time_point<_Clock, _Dur>& __atime) - { - static_assert(std::__is_one_of<_Clock, chrono::steady_clock, - chrono::system_clock>::value); - - auto __s = chrono::time_point_cast<chrono::seconds>(__atime); - auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s); - - __gthread_time_t __ts = - { - static_cast<std::time_t>(__s.time_since_epoch().count()), - static_cast<long>(__ns.count()) - }; +// (e.g. __ulock_wait) which is better than pthread_cond_clockwait. +#endif // ! HAVE_LINUX_FUTEX -#ifdef _GLIBCXX_USE_PTHREAD_COND_CLOCKWAIT - if constexpr (is_same_v<chrono::steady_clock, _Clock>) - __cv.wait_until(__mx, CLOCK_MONOTONIC, __ts); - else -#endif - __cv.wait_until(__mx, __ts); - return _Clock::now() < __atime; - } + __wait_result_type + __wait_until_impl(const void* __addr, __wait_args_base& __args, + const __wait_clock_t::duration& __atime); - // returns true if wait ended before timeout template<typename _Clock, typename _Dur> - bool - __cond_wait_until(__condvar& __cv, mutex& __mx, - const chrono::time_point<_Clock, _Dur>& __atime) + __wait_result_type + __wait_until(const void* __addr, __wait_args_base& __args, + const chrono::time_point<_Clock, _Dur>& __atime) noexcept { -#ifdef _GLIBCXX_USE_PTHREAD_COND_CLOCKWAIT - if constexpr (is_same_v<_Clock, chrono::steady_clock>) - return __detail::__cond_wait_until_impl(__cv, __mx, __atime); - else -#endif - if constexpr (is_same_v<_Clock, chrono::system_clock>) - return __detail::__cond_wait_until_impl(__cv, __mx, __atime); - else - { - if (__cond_wait_until_impl(__cv, __mx, - __to_wait_clock(__atime))) - { - // We got a timeout when measured against __clock_t but - // we need to check against the caller-supplied clock - // to tell whether we should return a timeout. - if (_Clock::now() < __atime) - return true; - } - return false; - } - } -#endif // _GLIBCXX_HAS_GTHREADS + auto __at = __detail::__to_wait_clock(__atime); + auto __res = __detail::__wait_until_impl(__addr, __args, + __at.time_since_epoch()); - struct __timed_waiter_pool : __waiter_pool_base - { - // returns true if wait ended before timeout - template<typename _Clock, typename _Dur> - bool - _M_do_wait_until(__platform_wait_t* __addr, __platform_wait_t __old, - const chrono::time_point<_Clock, _Dur>& __atime) - { -#ifdef _GLIBCXX_HAVE_PLATFORM_TIMED_WAIT - return __platform_wait_until(__addr, __old, __atime); -#else - __platform_wait_t __val; - __atomic_load(__addr, &__val, __ATOMIC_RELAXED); - if (__val == __old) + if constexpr (!is_same_v<__wait_clock_t, _Clock>) + if (__res._M_timeout) { - lock_guard<mutex> __l(_M_mtx); - return __cond_wait_until(_M_cv, _M_mtx, __atime); + // We got a timeout when measured against __clock_t but + // we need to check against the caller-supplied clock + // to tell whether we should return a timeout. + if (_Clock::now() < __atime) + __res._M_timeout = false; } - else - return true; -#endif // _GLIBCXX_HAVE_PLATFORM_TIMED_WAIT - } - }; - - struct __timed_backoff_spin_policy - { - __wait_clock_t::time_point _M_deadline; - __wait_clock_t::time_point _M_t0; - - template<typename _Clock, typename _Dur> - __timed_backoff_spin_policy(chrono::time_point<_Clock, _Dur> - __deadline = _Clock::time_point::max(), - chrono::time_point<_Clock, _Dur> - __t0 = _Clock::now()) noexcept - : _M_deadline(__to_wait_clock(__deadline)) - , _M_t0(__to_wait_clock(__t0)) - { } - - bool - operator()() const noexcept - { - using namespace literals::chrono_literals; - auto __now = __wait_clock_t::now(); - if (_M_deadline <= __now) - return false; - - // FIXME: this_thread::sleep_for not available #ifdef _GLIBCXX_NO_SLEEP - - auto __elapsed = __now - _M_t0; - if (__elapsed > 128ms) - { - this_thread::sleep_for(64ms); - } - else if (__elapsed > 64us) - { - this_thread::sleep_for(__elapsed / 2); - } - else if (__elapsed > 4us) - { - __thread_yield(); - } - else - return false; - return true; + return __res; } - }; - template<typename _EntersWait> - struct __timed_waiter : __waiter_base<__timed_waiter_pool> + template<typename _Rep, typename _Period> + __wait_result_type + __wait_for(const void* __addr, __wait_args_base& __args, + const chrono::duration<_Rep, _Period>& __rtime) noexcept { - using __base_type = __waiter_base<__timed_waiter_pool>; - - template<typename _Tp> - __timed_waiter(const _Tp* __addr) noexcept - : __base_type(__addr) - { - if constexpr (_EntersWait::value) - _M_w._M_enter_wait(); - } - - ~__timed_waiter() - { - if constexpr (_EntersWait::value) - _M_w._M_leave_wait(); - } - - // returns true if wait ended before timeout - template<typename _Tp, typename _ValFn, - typename _Clock, typename _Dur> - bool - _M_do_wait_until_v(_Tp __old, _ValFn __vfn, - const chrono::time_point<_Clock, _Dur>& - __atime) noexcept - { - __platform_wait_t __val; - if (_M_do_spin(__old, std::move(__vfn), __val, - __timed_backoff_spin_policy(__atime))) - return true; - return __base_type::_M_w._M_do_wait_until(__base_type::_M_addr, __val, __atime); - } - - // returns true if wait ended before timeout - template<typename _Pred, - typename _Clock, typename _Dur> - bool - _M_do_wait_until(_Pred __pred, __platform_wait_t __val, - const chrono::time_point<_Clock, _Dur>& - __atime) noexcept - { - for (auto __now = _Clock::now(); __now < __atime; - __now = _Clock::now()) - { - if (__base_type::_M_w._M_do_wait_until( - __base_type::_M_addr, __val, __atime) - && __pred()) - return true; - - if (__base_type::_M_do_spin(__pred, __val, - __timed_backoff_spin_policy(__atime, __now))) - return true; - } - return false; - } - - // returns true if wait ended before timeout - template<typename _Pred, - typename _Clock, typename _Dur> - bool - _M_do_wait_until(_Pred __pred, - const chrono::time_point<_Clock, _Dur>& - __atime) noexcept + if (!__rtime.count()) { - __platform_wait_t __val; - if (__base_type::_M_do_spin(__pred, __val, - __timed_backoff_spin_policy(__atime))) - return true; - return _M_do_wait_until(__pred, __val, __atime); - } - - template<typename _Tp, typename _ValFn, - typename _Rep, typename _Period> - bool - _M_do_wait_for_v(_Tp __old, _ValFn __vfn, - const chrono::duration<_Rep, _Period>& - __rtime) noexcept - { - __platform_wait_t __val; - if (_M_do_spin_v(__old, std::move(__vfn), __val)) - return true; - - if (!__rtime.count()) - return false; // no rtime supplied, and spin did not acquire - - auto __reltime = chrono::ceil<__wait_clock_t::duration>(__rtime); - - return __base_type::_M_w._M_do_wait_until( - __base_type::_M_addr, - __val, - chrono::steady_clock::now() + __reltime); + // no rtime supplied, just spin a bit + __args._M_flags |= __wait_flags::__do_spin | __wait_flags::__spin_only; + return __detail::__wait_impl(__addr, __args); } - template<typename _Pred, - typename _Rep, typename _Period> - bool - _M_do_wait_for(_Pred __pred, - const chrono::duration<_Rep, _Period>& __rtime) noexcept - { - __platform_wait_t __val; - if (__base_type::_M_do_spin(__pred, __val)) - return true; - - if (!__rtime.count()) - return false; // no rtime supplied, and spin did not acquire - - auto __reltime = chrono::ceil<__wait_clock_t::duration>(__rtime); - - return _M_do_wait_until(__pred, __val, - chrono::steady_clock::now() + __reltime); - } - }; - - using __enters_timed_wait = __timed_waiter<std::true_type>; - using __bare_timed_wait = __timed_waiter<std::false_type>; + auto const __reltime = chrono::ceil<__wait_clock_t::duration>(__rtime); + auto const __atime = chrono::steady_clock::now() + __reltime; + return __detail::__wait_until(__addr, __args, __atime); + } } // namespace __detail // returns true if wait ended before timeout - template<typename _Tp, typename _ValFn, + template<typename _Tp, + typename _Pred, typename _ValFn, typename _Clock, typename _Dur> bool - __atomic_wait_address_until_v(const _Tp* __addr, _Tp&& __old, _ValFn&& __vfn, - const chrono::time_point<_Clock, _Dur>& - __atime) noexcept + __atomic_wait_address_until(const _Tp* __addr, _Pred&& __pred, + _ValFn&& __vfn, + const chrono::time_point<_Clock, _Dur>& __atime, + bool __bare_wait = false) noexcept { - __detail::__enters_timed_wait __w{__addr}; - return __w._M_do_wait_until_v(__old, __vfn, __atime); + __detail::__wait_args __args{ __addr, __bare_wait }; + _Tp __val = __args._M_setup_wait(__addr, __vfn); + while (!__pred(__val)) + { + auto __res = __detail::__wait_until(__addr, __args, __atime); + if (__res._M_timeout) + return false; // C++26 will also return last observed __val + __val = __args._M_setup_wait(__addr, __vfn, __res); + } + return true; // C++26 will also return last observed __val } - template<typename _Tp, typename _Pred, - typename _Clock, typename _Dur> + template<typename _Clock, typename _Dur> bool - __atomic_wait_address_until(const _Tp* __addr, _Pred __pred, - const chrono::time_point<_Clock, _Dur>& - __atime) noexcept + __atomic_wait_address_until_v(const __detail::__platform_wait_t* __addr, + __detail::__platform_wait_t __old, + int __order, + const chrono::time_point<_Clock, _Dur>& __atime, + bool __bare_wait = false) noexcept { - __detail::__enters_timed_wait __w{__addr}; - return __w._M_do_wait_until(__pred, __atime); + __detail::__wait_args __args{ __addr, __old, __order, __bare_wait }; + auto __res = __detail::__wait_until(__addr, __args, __atime); + return !__res._M_timeout; // C++26 will also return last observed __val } - template<typename _Pred, + template<typename _Tp, typename _ValFn, typename _Clock, typename _Dur> bool - __atomic_wait_address_until_bare(const __detail::__platform_wait_t* __addr, - _Pred __pred, - const chrono::time_point<_Clock, _Dur>& - __atime) noexcept + __atomic_wait_address_until_v(const _Tp* __addr, _Tp&& __old, + _ValFn&& __vfn, + const chrono::time_point<_Clock, _Dur>& __atime, + bool __bare_wait = false) noexcept { - __detail::__bare_timed_wait __w{__addr}; - return __w._M_do_wait_until(__pred, __atime); + auto __pfn = [&](const _Tp& __val) { + return !__detail::__atomic_eq(__old, __val); + }; + return std::__atomic_wait_address_until(__addr, __pfn, __vfn, __atime, + __bare_wait); } - template<typename _Tp, typename _ValFn, + template<typename _Tp, + typename _Pred, typename _ValFn, typename _Rep, typename _Period> bool - __atomic_wait_address_for_v(const _Tp* __addr, _Tp&& __old, _ValFn&& __vfn, - const chrono::duration<_Rep, _Period>& __rtime) noexcept + __atomic_wait_address_for(const _Tp* __addr, _Pred&& __pred, + _ValFn&& __vfn, + const chrono::duration<_Rep, _Period>& __rtime, + bool __bare_wait = false) noexcept { - __detail::__enters_timed_wait __w{__addr}; - return __w._M_do_wait_for_v(__old, __vfn, __rtime); + __detail::__wait_args __args{ __addr, __bare_wait }; + _Tp __val = __args._M_setup_wait(__addr, __vfn); + while (!__pred(__val)) + { + auto __res = __detail::__wait_for(__addr, __args, __rtime); + if (__res._M_timeout) + return false; // C++26 will also return last observed __val + __val = __args._M_setup_wait(__addr, __vfn); + } + return true; // C++26 will also return last observed __val } - template<typename _Tp, typename _Pred, - typename _Rep, typename _Period> + template<typename _Rep, typename _Period> bool - __atomic_wait_address_for(const _Tp* __addr, _Pred __pred, - const chrono::duration<_Rep, _Period>& __rtime) noexcept + __atomic_wait_address_for_v(const __detail::__platform_wait_t* __addr, + __detail::__platform_wait_t __old, + int __order, + const chrono::duration<_Rep, _Period>& __rtime, + bool __bare_wait = false) noexcept { - - __detail::__enters_timed_wait __w{__addr}; - return __w._M_do_wait_for(__pred, __rtime); + __detail::__wait_args __args{ __addr, __old, __order, __bare_wait }; + auto __res = __detail::__wait_for(__addr, __args, __rtime); + return !__res._M_timeout; // C++26 will also return last observed __val } - template<typename _Pred, + template<typename _Tp, typename _ValFn, typename _Rep, typename _Period> bool - __atomic_wait_address_for_bare(const __detail::__platform_wait_t* __addr, - _Pred __pred, - const chrono::duration<_Rep, _Period>& __rtime) noexcept + __atomic_wait_address_for_v(const _Tp* __addr, _Tp&& __old, _ValFn&& __vfn, + const chrono::duration<_Rep, _Period>& __rtime, + bool __bare_wait = false) noexcept { - __detail::__bare_timed_wait __w{__addr}; - return __w._M_do_wait_for(__pred, __rtime); + auto __pfn = [&](const _Tp& __val) { + return !__detail::__atomic_eq(__old, __val); + }; + return __atomic_wait_address_for(__addr, __pfn, forward<_ValFn>(__vfn), + __rtime, __bare_wait); } _GLIBCXX_END_NAMESPACE_VERSION } // namespace std diff --git a/libstdc++-v3/include/bits/atomic_wait.h b/libstdc++-v3/include/bits/atomic_wait.h index 6d1554f..815726c 100644 --- a/libstdc++-v3/include/bits/atomic_wait.h +++ b/libstdc++-v3/include/bits/atomic_wait.h @@ -37,20 +37,10 @@ #include <bits/version.h> #if __glibcxx_atomic_wait -#include <cstdint> -#include <bits/functional_hash.h> #include <bits/gthr.h> #include <ext/numeric_traits.h> -#ifdef _GLIBCXX_HAVE_LINUX_FUTEX -# include <cerrno> -# include <climits> -# include <unistd.h> -# include <syscall.h> -# include <bits/functexcept.h> -#endif - -# include <bits/std_mutex.h> // std::mutex, std::__condvar +#include <bits/stl_pair.h> namespace std _GLIBCXX_VISIBILITY(default) { @@ -81,60 +71,18 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT = is_scalar_v<_Tp> && ((sizeof(_Tp) == sizeof(__detail::__platform_wait_t)) - && (alignof(_Tp*) >= __detail::__platform_wait_alignment)); + && (alignof(_Tp) >= __detail::__platform_wait_alignment)); #else = false; #endif namespace __detail { -#ifdef _GLIBCXX_HAVE_LINUX_FUTEX - enum class __futex_wait_flags : int - { -#ifdef _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE - __private_flag = 128, -#else - __private_flag = 0, -#endif - __wait = 0, - __wake = 1, - __wait_bitset = 9, - __wake_bitset = 10, - __wait_private = __wait | __private_flag, - __wake_private = __wake | __private_flag, - __wait_bitset_private = __wait_bitset | __private_flag, - __wake_bitset_private = __wake_bitset | __private_flag, - __bitset_match_any = -1 - }; - - template<typename _Tp> - void - __platform_wait(const _Tp* __addr, __platform_wait_t __val) noexcept - { - auto __e = syscall (SYS_futex, static_cast<const void*>(__addr), - static_cast<int>(__futex_wait_flags::__wait_private), - __val, nullptr); - if (!__e || errno == EAGAIN) - return; - if (errno != EINTR) - __throw_system_error(errno); - } - - template<typename _Tp> - void - __platform_notify(const _Tp* __addr, bool __all) noexcept - { - syscall (SYS_futex, static_cast<const void*>(__addr), - static_cast<int>(__futex_wait_flags::__wake_private), - __all ? INT_MAX : 1); - } -#endif - inline void __thread_yield() noexcept { #if defined _GLIBCXX_HAS_GTHREADS && defined _GLIBCXX_USE_SCHED_YIELD - __gthread_yield(); + __gthread_yield(); #endif } @@ -148,334 +96,189 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION #endif } - inline constexpr auto __atomic_spin_count_relax = 12; - inline constexpr auto __atomic_spin_count = 16; - - struct __default_spin_policy - { - bool - operator()() const noexcept - { return false; } - }; - - template<typename _Pred, - typename _Spin = __default_spin_policy> - bool - __atomic_spin(_Pred& __pred, _Spin __spin = _Spin{ }) noexcept - { - for (auto __i = 0; __i < __atomic_spin_count; ++__i) - { - if (__pred()) - return true; - - if (__i < __atomic_spin_count_relax) - __detail::__thread_relax(); - else - __detail::__thread_yield(); - } - - while (__spin()) - { - if (__pred()) - return true; - } - - return false; - } - // return true if equal template<typename _Tp> - bool __atomic_compare(const _Tp& __a, const _Tp& __b) + inline bool + __atomic_eq(const _Tp& __a, const _Tp& __b) { // TODO make this do the correct padding bit ignoring comparison return __builtin_memcmp(&__a, &__b, sizeof(_Tp)) == 0; } - struct __waiter_pool_base + // lightweight std::optional<__platform_wait_t> + struct __wait_result_type { - // Don't use std::hardware_destructive_interference_size here because we - // don't want the layout of library types to depend on compiler options. - static constexpr auto _S_align = 64; - - alignas(_S_align) __platform_wait_t _M_wait = 0; - -#ifndef _GLIBCXX_HAVE_PLATFORM_WAIT - mutex _M_mtx; -#endif + __platform_wait_t _M_val; + unsigned char _M_has_val : 1; // _M_val value was loaded before return. + unsigned char _M_timeout : 1; // Waiting function ended with timeout. + unsigned char _M_unused : 6; // padding + }; - alignas(_S_align) __platform_wait_t _M_ver = 0; + enum class __wait_flags : __UINT_LEAST32_TYPE__ + { + __abi_version = 0, + __proxy_wait = 1, + __track_contention = 2, + __do_spin = 4, + __spin_only = 8, // Ignored unless __do_spin is also set. + // __abi_version_mask = 0xffff0000, + }; -#ifndef _GLIBCXX_HAVE_PLATFORM_WAIT - __condvar _M_cv; -#endif - __waiter_pool_base() = default; + [[__gnu__::__always_inline__]] + constexpr __wait_flags + operator|(__wait_flags __l, __wait_flags __r) noexcept + { + using _Ut = underlying_type_t<__wait_flags>; + return static_cast<__wait_flags>(static_cast<_Ut>(__l) + | static_cast<_Ut>(__r)); + } - void - _M_enter_wait() noexcept - { __atomic_fetch_add(&_M_wait, 1, __ATOMIC_SEQ_CST); } + [[__gnu__::__always_inline__]] + constexpr __wait_flags& + operator|=(__wait_flags& __l, __wait_flags __r) noexcept + { return __l = __l | __r; } - void - _M_leave_wait() noexcept - { __atomic_fetch_sub(&_M_wait, 1, __ATOMIC_RELEASE); } + // Simple aggregate containing arguments used by implementation details. + struct __wait_args_base + { + __wait_flags _M_flags; + int _M_order = __ATOMIC_ACQUIRE; + __platform_wait_t _M_old = 0; + void* _M_wait_state = nullptr; + // Test whether _M_flags & __flags is non-zero. bool - _M_waiting() const noexcept + operator&(__wait_flags __flags) const noexcept { - __platform_wait_t __res; - __atomic_load(&_M_wait, &__res, __ATOMIC_SEQ_CST); - return __res != 0; + using _Ut = underlying_type_t<__wait_flags>; + return static_cast<_Ut>(_M_flags) & static_cast<_Ut>(__flags); } + }; - void - _M_notify(__platform_wait_t* __addr, [[maybe_unused]] bool __all, - bool __bare) noexcept - { -#ifdef _GLIBCXX_HAVE_PLATFORM_WAIT - if (__addr == &_M_ver) - { - __atomic_fetch_add(__addr, 1, __ATOMIC_SEQ_CST); - __all = true; - } - - if (__bare || _M_waiting()) - __platform_notify(__addr, __all); -#else + // Utility for populating a __wait_args_base structure. + struct __wait_args : __wait_args_base + { + template<typename _Tp> requires (!is_same_v<_Tp, __wait_args>) + explicit + __wait_args(const _Tp* __addr, bool __bare_wait = false) noexcept + : __wait_args_base{ _S_flags_for(__addr, __bare_wait) } + { } + + __wait_args(const __platform_wait_t* __addr, __platform_wait_t __old, + int __order, bool __bare_wait = false) noexcept + : __wait_args_base{ _S_flags_for(__addr, __bare_wait), __order, __old } + { } + + __wait_args(const __wait_args&) noexcept = default; + __wait_args& operator=(const __wait_args&) noexcept = default; + + template<typename _ValFn, + typename _Tp = decay_t<decltype(std::declval<_ValFn&>()())>> + _Tp + _M_setup_wait(const void* __addr, _ValFn __vfn, + __wait_result_type __res = {}) { - lock_guard<mutex> __l(_M_mtx); - __atomic_fetch_add(__addr, 1, __ATOMIC_RELAXED); + if constexpr (__platform_wait_uses_type<_Tp>) + { + // If the wait is not proxied, the value we check when waiting + // is the value of the atomic variable itself. + + if (__res._M_has_val) // The previous wait loaded a recent value. + { + _M_old = __res._M_val; + return __builtin_bit_cast(_Tp, __res._M_val); + } + else // Load the value from __vfn + { + _Tp __val = __vfn(); + _M_old = __builtin_bit_cast(__platform_wait_t, __val); + return __val; + } + } + else // It's a proxy wait and the proxy's _M_ver is used. + { + if (__res._M_has_val) // The previous wait loaded a recent value. + _M_old = __res._M_val; + else // Load _M_ver from the proxy (must happen before __vfn()). + _M_load_proxy_wait_val(__addr); + return __vfn(); + } } - if (__bare || _M_waiting()) - _M_cv.notify_all(); -#endif - } - - static __waiter_pool_base& - _S_for(const void* __addr) noexcept - { - constexpr __UINTPTR_TYPE__ __ct = 16; - static __waiter_pool_base __w[__ct]; - auto __key = ((__UINTPTR_TYPE__)__addr >> 2) % __ct; - return __w[__key]; - } - }; - struct __waiter_pool : __waiter_pool_base - { + private: + // Populates _M_wait_state and _M_old from the proxy for __addr. void - _M_do_wait(const __platform_wait_t* __addr, __platform_wait_t __old) noexcept - { -#ifdef _GLIBCXX_HAVE_PLATFORM_WAIT - __platform_wait(__addr, __old); -#else - __platform_wait_t __val; - __atomic_load(__addr, &__val, __ATOMIC_SEQ_CST); - if (__val == __old) - { - lock_guard<mutex> __l(_M_mtx); - __atomic_load(__addr, &__val, __ATOMIC_RELAXED); - if (__val == __old) - _M_cv.wait(_M_mtx); - } -#endif // __GLIBCXX_HAVE_PLATFORM_WAIT - } - }; + _M_load_proxy_wait_val(const void* __addr); - template<typename _Tp> - struct __waiter_base - { - using __waiter_type = _Tp; - - __waiter_type& _M_w; - __platform_wait_t* _M_addr; - - template<typename _Up> - static __platform_wait_t* - _S_wait_addr(const _Up* __a, __platform_wait_t* __b) - { - if constexpr (__platform_wait_uses_type<_Up>) - return reinterpret_cast<__platform_wait_t*>(const_cast<_Up*>(__a)); - else - return __b; - } - - static __waiter_type& - _S_for(const void* __addr) noexcept + template<typename _Tp> + static constexpr __wait_flags + _S_flags_for(const _Tp*, bool __bare_wait) noexcept { - static_assert(sizeof(__waiter_type) == sizeof(__waiter_pool_base)); - auto& res = __waiter_pool_base::_S_for(__addr); - return reinterpret_cast<__waiter_type&>(res); + using enum __wait_flags; + __wait_flags __res = __abi_version | __do_spin; + if (!__bare_wait) + __res |= __track_contention; + if constexpr (!__platform_wait_uses_type<_Tp>) + __res |= __proxy_wait; + return __res; } + }; - template<typename _Up> - explicit __waiter_base(const _Up* __addr) noexcept - : _M_w(_S_for(__addr)) - , _M_addr(_S_wait_addr(__addr, &_M_w._M_ver)) - { } - - void - _M_notify(bool __all, bool __bare = false) noexcept - { _M_w._M_notify(_M_addr, __all, __bare); } - - template<typename _Up, typename _ValFn, - typename _Spin = __default_spin_policy> - static bool - _S_do_spin_v(__platform_wait_t* __addr, - const _Up& __old, _ValFn __vfn, - __platform_wait_t& __val, - _Spin __spin = _Spin{ }) - { - auto const __pred = [=] - { return !__detail::__atomic_compare(__old, __vfn()); }; - - if constexpr (__platform_wait_uses_type<_Up>) - { - __builtin_memcpy(&__val, &__old, sizeof(__val)); - } - else - { - __atomic_load(__addr, &__val, __ATOMIC_ACQUIRE); - } - return __atomic_spin(__pred, __spin); - } - - template<typename _Up, typename _ValFn, - typename _Spin = __default_spin_policy> - bool - _M_do_spin_v(const _Up& __old, _ValFn __vfn, - __platform_wait_t& __val, - _Spin __spin = _Spin{ }) - { return _S_do_spin_v(_M_addr, __old, __vfn, __val, __spin); } - - template<typename _Pred, - typename _Spin = __default_spin_policy> - static bool - _S_do_spin(const __platform_wait_t* __addr, - _Pred __pred, - __platform_wait_t& __val, - _Spin __spin = _Spin{ }) - { - __atomic_load(__addr, &__val, __ATOMIC_ACQUIRE); - return __atomic_spin(__pred, __spin); - } - - template<typename _Pred, - typename _Spin = __default_spin_policy> - bool - _M_do_spin(_Pred __pred, __platform_wait_t& __val, - _Spin __spin = _Spin{ }) - { return _S_do_spin(_M_addr, __pred, __val, __spin); } - }; - - template<typename _EntersWait> - struct __waiter : __waiter_base<__waiter_pool> - { - using __base_type = __waiter_base<__waiter_pool>; + __wait_result_type + __wait_impl(const void* __addr, __wait_args_base&); - template<typename _Tp> - explicit __waiter(const _Tp* __addr) noexcept - : __base_type(__addr) - { - if constexpr (_EntersWait::value) - _M_w._M_enter_wait(); - } + void + __notify_impl(const void* __addr, bool __all, const __wait_args_base&); + } // namespace __detail - ~__waiter() + // Wait on __addr while __pred(__vfn()) is false. + // If __bare_wait is false, increment a counter while waiting. + // For callers that keep their own count of waiters, use __bare_wait=true. + template<typename _Tp, typename _Pred, typename _ValFn> + void + __atomic_wait_address(const _Tp* __addr, _Pred&& __pred, _ValFn&& __vfn, + bool __bare_wait = false) noexcept + { + __detail::__wait_args __args{ __addr, __bare_wait }; + _Tp __val = __args._M_setup_wait(__addr, __vfn); + while (!__pred(__val)) { - if constexpr (_EntersWait::value) - _M_w._M_leave_wait(); + auto __res = __detail::__wait_impl(__addr, __args); + __val = __args._M_setup_wait(__addr, __vfn, __res); } + // C++26 will return __val + } - template<typename _Tp, typename _ValFn> - void - _M_do_wait_v(_Tp __old, _ValFn __vfn) - { - do - { - __platform_wait_t __val; - if (__base_type::_M_do_spin_v(__old, __vfn, __val)) - return; - __base_type::_M_w._M_do_wait(__base_type::_M_addr, __val); - } - while (__detail::__atomic_compare(__old, __vfn())); - } - - template<typename _Pred> - void - _M_do_wait(_Pred __pred) noexcept - { - do - { - __platform_wait_t __val; - if (__base_type::_M_do_spin(__pred, __val)) - return; - __base_type::_M_w._M_do_wait(__base_type::_M_addr, __val); - } - while (!__pred()); - } - }; - - using __enters_wait = __waiter<std::true_type>; - using __bare_wait = __waiter<std::false_type>; - } // namespace __detail + inline void + __atomic_wait_address_v(const __detail::__platform_wait_t* __addr, + __detail::__platform_wait_t __old, + int __order) + { + __detail::__wait_args __args{ __addr, __old, __order }; + // C++26 will not ignore the return value here + __detail::__wait_impl(__addr, __args); + } + // Wait on __addr while __vfn() == __old is true. template<typename _Tp, typename _ValFn> void __atomic_wait_address_v(const _Tp* __addr, _Tp __old, _ValFn __vfn) noexcept { - __detail::__enters_wait __w(__addr); - __w._M_do_wait_v(__old, __vfn); - } - - template<typename _Tp, typename _Pred> - void - __atomic_wait_address(const _Tp* __addr, _Pred __pred) noexcept - { - __detail::__enters_wait __w(__addr); - __w._M_do_wait(__pred); - } - - // This call is to be used by atomic types which track contention externally - template<typename _Pred> - void - __atomic_wait_address_bare(const __detail::__platform_wait_t* __addr, - _Pred __pred) noexcept - { -#ifdef _GLIBCXX_HAVE_PLATFORM_WAIT - do - { - __detail::__platform_wait_t __val; - if (__detail::__bare_wait::_S_do_spin(__addr, __pred, __val)) - return; - __detail::__platform_wait(__addr, __val); - } - while (!__pred()); -#else // !_GLIBCXX_HAVE_PLATFORM_WAIT - __detail::__bare_wait __w(__addr); - __w._M_do_wait(__pred); -#endif + auto __pfn = [&](const _Tp& __val) + { return !__detail::__atomic_eq(__old, __val); }; + std::__atomic_wait_address(__addr, __pfn, forward<_ValFn>(__vfn)); } template<typename _Tp> void - __atomic_notify_address(const _Tp* __addr, bool __all) noexcept + __atomic_notify_address(const _Tp* __addr, bool __all, + bool __bare_wait = false) noexcept { - __detail::__bare_wait __w(__addr); - __w._M_notify(__all); + __detail::__wait_args __args{ __addr, __bare_wait }; + __detail::__notify_impl(__addr, __all, __args); } - // This call is to be used by atomic types which track contention externally - inline void - __atomic_notify_address_bare(const __detail::__platform_wait_t* __addr, - bool __all) noexcept - { -#ifdef _GLIBCXX_HAVE_PLATFORM_WAIT - __detail::__platform_notify(__addr, __all); -#else - __detail::__bare_wait __w(__addr); - __w._M_notify(__all, true); -#endif - } _GLIBCXX_END_NAMESPACE_VERSION } // namespace std #endif // __glibcxx_atomic_wait diff --git a/libstdc++-v3/include/bits/basic_string.h b/libstdc++-v3/include/bits/basic_string.h index a087e63..7081049 100644 --- a/libstdc++-v3/include/bits/basic_string.h +++ b/libstdc++-v3/include/bits/basic_string.h @@ -1164,7 +1164,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CXX11 { size_type __sz = _M_string_length; if (__sz > max_size ()) - __builtin_unreachable (); + __builtin_unreachable(); return __sz; } @@ -1279,7 +1279,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CXX11 size_t __sz = _M_is_local() ? size_type(_S_local_capacity) : _M_allocated_capacity; if (__sz < _S_local_capacity || __sz > max_size ()) - __builtin_unreachable (); + __builtin_unreachable(); return __sz; } diff --git a/libstdc++-v3/include/bits/boost_concept_check.h b/libstdc++-v3/include/bits/boost_concept_check.h index 7a99f74..a1f488d 100644 --- a/libstdc++-v3/include/bits/boost_concept_check.h +++ b/libstdc++-v3/include/bits/boost_concept_check.h @@ -68,6 +68,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-local-typedefs" +#pragma GCC diagnostic ignored "-Wlong-long" #define _IsUnused __attribute__ ((__unused__)) diff --git a/libstdc++-v3/include/bits/cpyfunc_impl.h b/libstdc++-v3/include/bits/cpyfunc_impl.h index bc44cd3e..f1918dd 100644 --- a/libstdc++-v3/include/bits/cpyfunc_impl.h +++ b/libstdc++-v3/include/bits/cpyfunc_impl.h @@ -64,6 +64,10 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION _GLIBCXX_MOF_REF noexcept(_Noex)> : __polyfunc::_Cpy_base { + static_assert( + (std::__is_complete_or_unbounded(__type_identity<_ArgTypes>()) && ...), + "each parameter type must be a complete class"); + using _Base = __polyfunc::_Cpy_base; using _Invoker = __polyfunc::_Invoker<_Noex, _Res, _ArgTypes...>; using _Signature = _Invoker::_Signature; diff --git a/libstdc++-v3/include/bits/funcref_impl.h b/libstdc++-v3/include/bits/funcref_impl.h index 1e19866..44c9922 100644 --- a/libstdc++-v3/include/bits/funcref_impl.h +++ b/libstdc++-v3/include/bits/funcref_impl.h @@ -68,6 +68,10 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION class function_ref<_Res(_ArgTypes...) _GLIBCXX_MOF_CV noexcept(_Noex)> { + static_assert( + (std::__is_complete_or_unbounded(__type_identity<_ArgTypes>()) && ...), + "each parameter type must be a complete class"); + using _Invoker = __polyfunc::_Invoker<_Noex, _Res, _ArgTypes...>; using _Signature = _Invoker::_Signature; diff --git a/libstdc++-v3/include/bits/funcwrap.h b/libstdc++-v3/include/bits/funcwrap.h index cf261bc..9db4ab7 100644 --- a/libstdc++-v3/include/bits/funcwrap.h +++ b/libstdc++-v3/include/bits/funcwrap.h @@ -199,7 +199,25 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION }; template<typename _Tp> - using __param_t = __conditional_t<is_scalar_v<_Tp>, _Tp, _Tp&&>; + consteval bool + __pass_by_value() + { + // n.b. sizeof(Incomplete&) is ill-formed for incomplete types, + // so we check is_reference_v first. + if constexpr (is_reference_v<_Tp> || is_scalar_v<_Tp>) + return true; + else + // n.b. we already asserted that types are complete in wrappers, + // avoid triggering additional errors from this function. + if constexpr (std::__is_complete_or_unbounded(__type_identity<_Tp>())) + if constexpr (sizeof(_Tp) <= 2 * sizeof(void*)) + return is_trivially_move_constructible_v<_Tp> + && is_trivially_destructible_v<_Tp>; + return false; + } + + template<typename _Tp> + using __param_t = __conditional_t<__pass_by_value<_Tp>(), _Tp, _Tp&&>; template<bool _Noex, typename _Ret, typename... _Args> using _Invoker = _Base_invoker<_Noex, remove_cv_t<_Ret>, __param_t<_Args>...>; diff --git a/libstdc++-v3/include/bits/indirect.h b/libstdc++-v3/include/bits/indirect.h index 85908e2..e8000d7 100644 --- a/libstdc++-v3/include/bits/indirect.h +++ b/libstdc++-v3/include/bits/indirect.h @@ -452,7 +452,381 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION { }; #endif // __glibcxx_indirect - _GLIBCXX_END_NAMESPACE_VERSION +#if __glibcxx_polymorphic // C++26 && HOSTED + template<typename _Tp, typename _Alloc = allocator<_Tp>> + class polymorphic; + + namespace pmr + { + template<typename _Tp> + using polymorphic = polymorphic<_Tp, polymorphic_allocator<_Tp>>; + } + + // [polymorphic], class template polymorphic + template<typename _Tp, typename _Alloc> + class polymorphic + { + static_assert(is_object_v<_Tp>); + static_assert(!is_array_v<_Tp>); + static_assert(!is_same_v<_Tp, in_place_t>); + static_assert(!__is_in_place_type_v<_Tp>); + static_assert(!is_const_v<_Tp> && !is_volatile_v<_Tp>); + + using _ATraits = allocator_traits<_Alloc>; + static_assert(is_same_v<_Tp, typename _ATraits::value_type>); + + // The owned object is embedded within a control block which knows the + // dynamic type and manages cloning and destroying the owned object. + struct _Obj + { + typename _ATraits::pointer _M_objp{}; // pointer to the owned object. + + // A pointer to this type, e.g. _Obj* + using pointer + = typename _ATraits::template rebind_traits<_Obj>::pointer; + + enum class _Op { _Dispose = 1, _Copy = 2, _Move = 3 }; + + constexpr virtual pointer + _M_manage(const _Alloc&, _Op, void* = nullptr) = 0; + }; + + template<typename _Up> + struct _Obj_impl : _Obj + { + using _MyTraits + = typename _ATraits::template rebind_traits<_Obj_impl>; + + using _Op = _Obj::_Op; + + union _Uninitialized { + constexpr _Uninitialized() { } + constexpr ~_Uninitialized() { } + _Up _M_objp; + }; + _Uninitialized _M_u; + + template<typename... _Args> + constexpr + _Obj_impl(typename _MyTraits::allocator_type& __a, + _Args&&... __args) + { + using _PtrTr = pointer_traits<typename _ATraits::pointer>; + _MyTraits::construct(__a, __builtin_addressof(_M_u._M_objp), + std::forward<_Args>(__args)...); + this->_M_objp = _PtrTr::pointer_to(_M_u._M_objp); + } + + constexpr virtual typename _Obj::pointer + _M_manage(const _Alloc& __a, _Op __op, void*) override + { + + switch (__op) + { + case _Op::_Move: + return _S_make_obj<_Up>(__a, std::move(_M_u._M_objp)); + case _Op::_Copy: + return _S_make_obj<_Up>(__a, + const_cast<const _Up&>(_M_u._M_objp)); + case _Op::_Dispose: + { + using _PtrTr = pointer_traits<typename _MyTraits::pointer>; + typename _MyTraits::allocator_type __a2(__a); + _MyTraits::destroy(__a2, std::__addressof(_M_u._M_objp)); + _MyTraits::deallocate(__a2, _PtrTr::pointer_to(*this), 1); + return nullptr; + } + } + __builtin_unreachable(); + } + }; + + // TODO: the standard permits a small-object optimization where the + // owned object is nested within the std::polymorphic not on the heap. + + public: + + using value_type = _Tp; + using allocator_type = _Alloc; + using pointer = typename allocator_traits<_Alloc>::pointer; + using const_pointer = typename allocator_traits<_Alloc>::const_pointer; + + constexpr explicit + polymorphic() requires is_default_constructible_v<_Alloc> + : polymorphic(in_place_type<_Tp>) + { } + + constexpr explicit + polymorphic(allocator_arg_t, const _Alloc& __a) + : polymorphic(allocator_arg, __a, in_place_type<_Tp>) + { } + + constexpr + polymorphic(const polymorphic& __other) + : polymorphic(allocator_arg, + _ATraits::select_on_container_copy_construction( + __other._M_alloc), + __other) + { } + + constexpr + polymorphic(allocator_arg_t, const _Alloc& __a, + const polymorphic& __other) + : _M_alloc(__a) + { + if (__other._M_objp) + _M_objp = __other._M_objp->_M_manage(__a, _Obj::_Op::_Copy); + else + _M_objp = nullptr; + } + + constexpr + polymorphic(polymorphic&& __other) noexcept + : _M_alloc(std::move(__other._M_alloc)), + _M_objp(std::__exchange(__other._M_objp, nullptr)) + { } + + constexpr + polymorphic(allocator_arg_t, const _Alloc& __a, polymorphic&& __other) + noexcept(_ATraits::is_always_equal::value) + : _M_alloc(__a), + _M_objp(std::__exchange(__other._M_objp, nullptr)) + { + if constexpr (!_ATraits::is_always_equal::value) + if (_M_objp && _M_alloc != __other._M_alloc) + { + // _M_alloc cannot free _M_objp, give it back to __other. + __other._M_objp = std::__exchange(_M_objp, nullptr); + // And create a new object that can be freed by _M_alloc. + _M_objp = __other._M_objp->_M_manage(__a, _Obj::_Op::_Move); + } + } + + template<typename _Up = _Tp, typename _UUp = remove_cvref_t<_Up>> + requires (!is_same_v<_UUp, polymorphic>) + && (!__is_in_place_type_v<_UUp>) + && derived_from<_UUp, _Tp> + && is_constructible_v<_UUp, _Up> + && is_copy_constructible_v<_UUp> + && is_default_constructible_v<_Alloc> + constexpr explicit + polymorphic(_Up&& __u) + : _M_objp(_M_make_obj<_UUp>(std::forward<_Up>(__u))) + { } + + template<typename _Up = _Tp, typename _UUp = remove_cvref_t<_Up>> + requires (!is_same_v<_UUp, polymorphic>) + && (!__is_in_place_type_v<_UUp>) + && derived_from<_UUp, _Tp> + && is_constructible_v<_UUp, _Up> + && is_copy_constructible_v<_UUp> + constexpr explicit + polymorphic(allocator_arg_t, const _Alloc& __a, _Up&& __u) + : _M_alloc(__a), _M_objp(_M_make_obj<_UUp>(std::forward<_Up>(__u))) + { } + + template<typename _Up, typename... _Ts> + requires is_same_v<remove_cvref_t<_Up>, _Up> + && derived_from<_Up, _Tp> + && is_constructible_v<_Up, _Ts...> + && is_copy_constructible_v<_Up> + && is_default_constructible_v<_Alloc> + constexpr explicit + polymorphic(in_place_type_t<_Up> __t, _Ts&&... __ts) + : _M_objp(_M_make_obj<_Up>(std::forward<_Ts>(__ts)...)) + { } + + template<typename _Up, typename... _Ts> + requires is_same_v<remove_cvref_t<_Up>, _Up> + && derived_from<_Up, _Tp> + && is_constructible_v<_Up, _Ts...> + && is_copy_constructible_v<_Up> + constexpr explicit + polymorphic(allocator_arg_t, const _Alloc& __a, + in_place_type_t<_Up>, _Ts&&... __ts) + : _M_alloc(__a), + _M_objp(_M_make_obj<_Up>(std::forward<_Ts>(__ts)...)) + { } + + template<typename _Up, typename _Ip, typename... _Us> + requires is_same_v<remove_cvref_t<_Up>, _Up> + && derived_from<_Up, _Tp> + && is_constructible_v<_Up, initializer_list<_Ip>&, _Us...> + && is_copy_constructible_v<_Up> + && is_default_constructible_v<_Alloc> + constexpr explicit + polymorphic(in_place_type_t<_Up>, initializer_list<_Ip> __il, + _Us&&... __us) + : _M_objp(_M_make_obj<_Up>(__il, std::forward<_Us>(__us)...)) + { } + + template<typename _Up, typename _Ip, typename... _Us> + requires is_same_v<remove_cvref_t<_Up>, _Up> + && derived_from<_Up, _Tp> + && is_constructible_v<_Up, initializer_list<_Ip>&, _Us...> + && is_copy_constructible_v<_Up> + constexpr explicit + polymorphic(allocator_arg_t, const _Alloc& __a, + in_place_type_t<_Up>, initializer_list<_Ip> __il, + _Us&&... __us) + : _M_alloc(__a), + _M_objp(_M_make_obj<_Up>(__il, std::forward<_Us>(__us)...)) + { } + + constexpr ~polymorphic() + { + static_assert(sizeof(_Tp) != 0, "must be a complete type"); + _M_reset(nullptr); + } + + constexpr polymorphic& + operator=(const polymorphic& __other) + { + static_assert(sizeof(_Tp) != 0, "must be a complete type"); + + if (__builtin_addressof(__other) == this) [[unlikely]] + return *this; + + constexpr bool __pocca + = _ATraits::propagate_on_container_copy_assignment::value; + + typename _Obj::pointer __ptr = nullptr; + if (__other._M_objp) + { + auto& __a = __pocca ? __other._M_alloc : _M_alloc; + __ptr = __other._M_objp->_M_manage(__a, _Obj::_Op::_Copy); + } + + _M_reset(__ptr); + + if constexpr (__pocca) + _M_alloc = __other._M_alloc; + + return *this; + } + + constexpr polymorphic& + operator=(polymorphic&& __other) + noexcept(_ATraits::propagate_on_container_move_assignment::value + || _ATraits::is_always_equal::value) + { + if (__builtin_addressof(__other) == this) [[unlikely]] + return *this; + + constexpr bool __pocma + = _ATraits::propagate_on_container_move_assignment::value; + + typename _Obj::pointer __ptr = nullptr; + + // _GLIBCXX_RESOLVE_LIB_DEFECTS + // 4251. Move assignment for indirect unnecessarily requires copy construction + if constexpr (_ATraits::is_always_equal::value || __pocma) + __ptr = std::__exchange(__other._M_objp, nullptr); + else if (_M_alloc == __other._M_alloc) + __ptr = std::__exchange(__other._M_objp, nullptr); + else if (__other._M_objp) + { + static_assert(sizeof(_Tp) != 0, "must be a complete type"); + __ptr = __other._M_objp->_M_manage(_M_alloc, _Obj::_Op::_Move); + } + + _M_reset(__ptr); + + if constexpr (__pocma) + _M_alloc = __other._M_alloc; + + return *this; + } + + constexpr const _Tp& + operator*() const noexcept + { + __glibcxx_assert(_M_objp != nullptr); + return *_M_objp->_M_objp; + } + + constexpr _Tp& + operator*() noexcept + { + __glibcxx_assert(_M_objp != nullptr); + return *_M_objp->_M_objp; + } + + constexpr const_pointer + operator->() const noexcept + { + __glibcxx_assert(_M_objp != nullptr); + return _M_objp ? _M_objp->_M_objp : const_pointer{}; + } + + constexpr pointer + operator->() noexcept + { + __glibcxx_assert(_M_objp != nullptr); + return _M_objp ? _M_objp->_M_objp : pointer{}; + } + + constexpr bool + valueless_after_move() const noexcept { return _M_objp == nullptr; } + + constexpr allocator_type + get_allocator() const noexcept { return _M_alloc; } + + constexpr void + swap(polymorphic& __other) + noexcept(_ATraits::propagate_on_container_swap::value + || _ATraits::is_always_equal::value) + { + using std::swap; + swap(_M_objp, __other._M_objp); + if constexpr (_ATraits::propagate_on_container_swap::value) + swap(_M_alloc, __other._M_alloc); + else if constexpr (!_ATraits::is_always_equal::value) + __glibcxx_assert(_M_alloc == __other._M_alloc); + } + + friend constexpr void + swap(polymorphic& __lhs, polymorphic& __rhs) + noexcept(_ATraits::propagate_on_container_swap::value + || _ATraits::is_always_equal::value) + { __lhs.swap(__rhs); } + + private: + template<typename _Up, typename... _Args> + static constexpr typename _Obj::pointer + _S_make_obj(const _Alloc& __a, _Args&&... __args) + { + __alloc_rebind<_Alloc, _Obj_impl<_Up>> __objalloc(__a); + _Scoped_allocation __sa(__objalloc, in_place, __objalloc, + std::forward<_Args>(__args)...); + auto __obj = __sa.release(); + // FIXME: We need to downcast from _Obj_impl<U>* to _Obj* but the + // the pointer_traits usage breaks in constexpr. PR c++/110714 + if constexpr (is_pointer_v<typename _Obj::pointer>) + return __obj; + else + return pointer_traits<typename _Obj::pointer>::pointer_to(*__obj); + } + + template<typename _Up, typename... _Args> + constexpr typename _Obj::pointer + _M_make_obj(_Args&&... __args) const + { return _S_make_obj<_Up>(_M_alloc, std::forward<_Args>(__args)...); } + + constexpr void + _M_reset(typename _Obj::pointer __ptr) noexcept + { + if (_M_objp) + _M_objp->_M_manage(_M_alloc, _Obj::_Op::_Dispose); + _M_objp = __ptr; + } + + [[no_unique_address]] _Alloc _M_alloc = _Alloc(); + typename _Obj::pointer _M_objp; + }; +#endif // __glibcxx_polymorphic + +_GLIBCXX_END_NAMESPACE_VERSION } // namespace #endif // C++26 __glibcxx_indirect || __glibcxx_polymorphic diff --git a/libstdc++-v3/include/bits/mofunc_impl.h b/libstdc++-v3/include/bits/mofunc_impl.h index 1ceb910..468e685 100644 --- a/libstdc++-v3/include/bits/mofunc_impl.h +++ b/libstdc++-v3/include/bits/mofunc_impl.h @@ -64,6 +64,10 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION _GLIBCXX_MOF_REF noexcept(_Noex)> : __polyfunc::_Mo_base { + static_assert( + (std::__is_complete_or_unbounded(__type_identity<_ArgTypes>()) && ...), + "each parameter type must be a complete class"); + using _Base = __polyfunc::_Mo_base; using _Invoker = __polyfunc::_Invoker<_Noex, _Res, _ArgTypes...>; using _Signature = _Invoker::_Signature; diff --git a/libstdc++-v3/include/bits/ranges_algo.h b/libstdc++-v3/include/bits/ranges_algo.h index f36e7dd..7b14084 100644 --- a/libstdc++-v3/include/bits/ranges_algo.h +++ b/libstdc++-v3/include/bits/ranges_algo.h @@ -1218,6 +1218,9 @@ namespace ranges if (__first == __last) return {std::move(__first), std::move(__result)}; + // _GLIBCXX_RESOLVE_LIB_DEFECTS + // 4269. unique_copy passes arguments to its predicate backwards + // TODO: perform a closer comparison with reference implementations if constexpr (forward_iterator<_Iter>) { @@ -1250,8 +1253,8 @@ namespace ranges while (++__first != __last) { if (!(bool)std::__invoke(__comp, - std::__invoke(__proj, *__first), - std::__invoke(__proj, __value))) + std::__invoke(__proj, __value), + std::__invoke(__proj, *__first))) { __value = *__first; *++__result = __value; diff --git a/libstdc++-v3/include/bits/semaphore_base.h b/libstdc++-v3/include/bits/semaphore_base.h index d8f9bd8..5b5a1c9 100644 --- a/libstdc++-v3/include/bits/semaphore_base.h +++ b/libstdc++-v3/include/bits/semaphore_base.h @@ -34,157 +34,45 @@ #pragma GCC system_header #endif +#include <bits/version.h> + +#ifdef __glibcxx_semaphore // C++ >= 20 && hosted && atomic_wait #include <bits/atomic_base.h> #include <bits/chrono.h> -#if __glibcxx_atomic_wait #include <bits/atomic_timed_wait.h> #include <ext/numeric_traits.h> -#endif // __cpp_lib_atomic_wait - -#ifdef _GLIBCXX_HAVE_POSIX_SEMAPHORE -# include <cerrno> // errno, EINTR, EAGAIN etc. -# include <limits.h> // SEM_VALUE_MAX -# include <semaphore.h> // sem_t, sem_init, sem_wait, sem_post etc. -#elif defined(_GLIBCXX_USE_POSIX_SEMAPHORE) -# warning "POSIX semaphore not available, ignoring _GLIBCXX_USE_POSIX_SEMAPHORE" -# undef _GLIBCXX_USE_POSIX_SEMAPHORE -#endif namespace std _GLIBCXX_VISIBILITY(default) { _GLIBCXX_BEGIN_NAMESPACE_VERSION -#ifdef _GLIBCXX_HAVE_POSIX_SEMAPHORE - struct __platform_semaphore + template<bool _Platform_wait> + struct __semaphore_base { - using __clock_t = chrono::system_clock; -#ifdef SEM_VALUE_MAX - static constexpr ptrdiff_t _S_max = SEM_VALUE_MAX; -#else - static constexpr ptrdiff_t _S_max = _POSIX_SEM_VALUE_MAX; -#endif - - explicit __platform_semaphore(ptrdiff_t __count) noexcept - { - sem_init(&_M_semaphore, 0, __count); - } - - __platform_semaphore(const __platform_semaphore&) = delete; - __platform_semaphore& operator=(const __platform_semaphore&) = delete; - - ~__platform_semaphore() - { sem_destroy(&_M_semaphore); } - - _GLIBCXX_ALWAYS_INLINE void - _M_acquire() noexcept - { - while (sem_wait(&_M_semaphore)) - if (errno != EINTR) - std::__terminate(); - } - - _GLIBCXX_ALWAYS_INLINE bool - _M_try_acquire() noexcept - { - while (sem_trywait(&_M_semaphore)) - { - if (errno == EAGAIN) // already locked - return false; - else if (errno != EINTR) - std::__terminate(); - // else got EINTR so retry - } - return true; - } - - _GLIBCXX_ALWAYS_INLINE void - _M_release(ptrdiff_t __update) noexcept - { - for(; __update != 0; --__update) - if (sem_post(&_M_semaphore)) - std::__terminate(); - } - - bool - _M_try_acquire_until_impl(const chrono::time_point<__clock_t>& __atime) - noexcept - { - auto __s = chrono::time_point_cast<chrono::seconds>(__atime); - auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s); - - struct timespec __ts = - { - static_cast<std::time_t>(__s.time_since_epoch().count()), - static_cast<long>(__ns.count()) - }; - - while (sem_timedwait(&_M_semaphore, &__ts)) - { - if (errno == ETIMEDOUT) - return false; - else if (errno != EINTR) - std::__terminate(); - } - return true; - } - - template<typename _Clock, typename _Duration> - bool - _M_try_acquire_until(const chrono::time_point<_Clock, - _Duration>& __atime) noexcept - { - if constexpr (std::is_same_v<__clock_t, _Clock>) - { - using _Dur = __clock_t::duration; - return _M_try_acquire_until_impl(chrono::ceil<_Dur>(__atime)); - } - else - { - // TODO: if _Clock is monotonic_clock we could use - // sem_clockwait with CLOCK_MONOTONIC. - - const typename _Clock::time_point __c_entry = _Clock::now(); - const auto __s_entry = __clock_t::now(); - const auto __delta = __atime - __c_entry; - const auto __s_atime = __s_entry + __delta; - if (_M_try_acquire_until_impl(__s_atime)) - return true; + using __count_type = __conditional_t<_Platform_wait, + __detail::__platform_wait_t, + ptrdiff_t>; - // We got a timeout when measured against __clock_t but - // we need to check against the caller-supplied clock - // to tell whether we should return a timeout. - return (_Clock::now() < __atime); - } - } + static constexpr ptrdiff_t _S_max + = __gnu_cxx::__int_traits<__count_type>::__max; - template<typename _Rep, typename _Period> - _GLIBCXX_ALWAYS_INLINE bool - _M_try_acquire_for(const chrono::duration<_Rep, _Period>& __rtime) - noexcept - { return _M_try_acquire_until(__clock_t::now() + __rtime); } + constexpr explicit + __semaphore_base(__count_type __count) noexcept + : _M_counter(__count) + { } - private: - sem_t _M_semaphore; - }; -#endif // _GLIBCXX_HAVE_POSIX_SEMAPHORE + __semaphore_base(const __semaphore_base&) = delete; + __semaphore_base& operator=(const __semaphore_base&) = delete; -#if __glibcxx_atomic_wait - struct __atomic_semaphore - { - static constexpr ptrdiff_t _S_max = __gnu_cxx::__int_traits<int>::__max; - explicit __atomic_semaphore(__detail::__platform_wait_t __count) noexcept - : _M_counter(__count) + static _GLIBCXX_ALWAYS_INLINE __count_type + _S_get_current(__count_type* __counter) noexcept { - __glibcxx_assert(__count >= 0 && __count <= _S_max); + return __atomic_impl::load(__counter, memory_order::acquire); } - __atomic_semaphore(const __atomic_semaphore&) = delete; - __atomic_semaphore& operator=(const __atomic_semaphore&) = delete; - static _GLIBCXX_ALWAYS_INLINE bool - _S_do_try_acquire(__detail::__platform_wait_t* __counter) noexcept + _S_do_try_acquire(__count_type* __counter, __count_type __old) noexcept { - auto __old = __atomic_impl::load(__counter, memory_order::acquire); if (__old == 0) return false; @@ -197,68 +85,71 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION _GLIBCXX_ALWAYS_INLINE void _M_acquire() noexcept { - auto const __pred = - [this] { return _S_do_try_acquire(&this->_M_counter); }; - std::__atomic_wait_address_bare(&_M_counter, __pred); + auto const __vfn = [this]{ return _S_get_current(&this->_M_counter); }; + auto const __pred = [this](__count_type __cur) { + return _S_do_try_acquire(&this->_M_counter, __cur); + }; + std::__atomic_wait_address(&_M_counter, __pred, __vfn, true); } bool _M_try_acquire() noexcept { - auto const __pred = - [this] { return _S_do_try_acquire(&this->_M_counter); }; - return std::__detail::__atomic_spin(__pred); + auto const __vfn = [this]{ return _S_get_current(&this->_M_counter); }; + auto const __pred = [this](__count_type __cur) { + return _S_do_try_acquire(&this->_M_counter, __cur); + }; + using __detail::__wait_clock_t; + return std::__atomic_wait_address_for(&_M_counter, __pred, __vfn, + __wait_clock_t::duration(), + true); } template<typename _Clock, typename _Duration> _GLIBCXX_ALWAYS_INLINE bool - _M_try_acquire_until(const chrono::time_point<_Clock, - _Duration>& __atime) noexcept + _M_try_acquire_until(const chrono::time_point<_Clock, _Duration>& __atime) noexcept { - auto const __pred = - [this] { return _S_do_try_acquire(&this->_M_counter); }; - - return __atomic_wait_address_until_bare(&_M_counter, __pred, __atime); + auto const __vfn = [this]{ return _S_get_current(&this->_M_counter); }; + auto const __pred = [this](__count_type __cur) { + return _S_do_try_acquire(&this->_M_counter, __cur); + }; + return std::__atomic_wait_address_until(&_M_counter, __pred, __vfn, + __atime, true); } template<typename _Rep, typename _Period> _GLIBCXX_ALWAYS_INLINE bool - _M_try_acquire_for(const chrono::duration<_Rep, _Period>& __rtime) - noexcept + _M_try_acquire_for(const chrono::duration<_Rep, _Period>& __rtime) noexcept { - auto const __pred = - [this] { return _S_do_try_acquire(&this->_M_counter); }; - - return __atomic_wait_address_for_bare(&_M_counter, __pred, __rtime); + auto const __vfn = [this]{ return _S_get_current(&this->_M_counter); }; + auto const __pred = [this](__count_type __cur) { + return _S_do_try_acquire(&this->_M_counter, __cur); + }; + return std::__atomic_wait_address_for(&_M_counter, __pred, __vfn, + __rtime, true); } - _GLIBCXX_ALWAYS_INLINE void + _GLIBCXX_ALWAYS_INLINE ptrdiff_t _M_release(ptrdiff_t __update) noexcept { - if (0 < __atomic_impl::fetch_add(&_M_counter, __update, memory_order_release)) - return; - if (__update > 1) - __atomic_notify_address_bare(&_M_counter, true); - else - __atomic_notify_address_bare(&_M_counter, true); -// FIXME - Figure out why this does not wake a waiting thread -// __atomic_notify_address_bare(&_M_counter, false); + auto __old = __atomic_impl::fetch_add(&_M_counter, __update, + memory_order::release); + if (__old == 0 && __update > 0) + __atomic_notify_address(&_M_counter, true, true); + return __old; } private: - alignas(__detail::__platform_wait_alignment) - __detail::__platform_wait_t _M_counter; + alignas(_Platform_wait ? __detail::__platform_wait_alignment + : __alignof__(__count_type)) + __count_type _M_counter; }; -#endif // __cpp_lib_atomic_wait -// Note: the _GLIBCXX_USE_POSIX_SEMAPHORE macro can be used to force the -// use of Posix semaphores (sem_t). Doing so however, alters the ABI. -#if defined __glibcxx_atomic_wait && !_GLIBCXX_USE_POSIX_SEMAPHORE - using __semaphore_impl = __atomic_semaphore; -#elif _GLIBCXX_HAVE_POSIX_SEMAPHORE - using __semaphore_impl = __platform_semaphore; -#endif + template<ptrdiff_t _Max> + using __semaphore_impl + = __semaphore_base<(_Max <= __semaphore_base<true>::_S_max)>; _GLIBCXX_END_NAMESPACE_VERSION } // namespace std +#endif // __glibcxx_semaphore #endif // _GLIBCXX_SEMAPHORE_BASE_H diff --git a/libstdc++-v3/include/bits/stl_algo.h b/libstdc++-v3/include/bits/stl_algo.h index f5361ae..98c2249 100644 --- a/libstdc++-v3/include/bits/stl_algo.h +++ b/libstdc++-v3/include/bits/stl_algo.h @@ -918,52 +918,45 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION __gnu_cxx::__ops::__iter_comp_iter(__binary_pred)); } - /** - * This is an uglified - * unique_copy(_InputIterator, _InputIterator, _OutputIterator, - * _BinaryPredicate) - * overloaded for forward iterators and output iterator as result. - */ + // _GLIBCXX_RESOLVE_LIB_DEFECTS + // 4269. unique_copy passes arguments to its predicate backwards + + // Implementation of std::unique_copy for forward iterators. + // This case is easy, just compare *i with *(i-1). template<typename _ForwardIterator, typename _OutputIterator, typename _BinaryPredicate> _GLIBCXX20_CONSTEXPR _OutputIterator __unique_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, _BinaryPredicate __binary_pred, - forward_iterator_tag, output_iterator_tag) + forward_iterator_tag) { - _ForwardIterator __next = __first; + _ForwardIterator __prev = __first; *__result = *__first; - while (++__next != __last) - if (!__binary_pred(__first, __next)) + while (++__first != __last) + if (!__binary_pred(__prev, __first)) { - __first = __next; *++__result = *__first; + __prev = __first; } return ++__result; } - /** - * This is an uglified - * unique_copy(_InputIterator, _InputIterator, _OutputIterator, - * _BinaryPredicate) - * overloaded for input iterators and output iterator as result. - */ + // Implementation of std::unique_copy for non-forward iterators, + // where we cannot compare with elements written to the output. template<typename _InputIterator, typename _OutputIterator, typename _BinaryPredicate> _GLIBCXX20_CONSTEXPR _OutputIterator - __unique_copy(_InputIterator __first, _InputIterator __last, - _OutputIterator __result, _BinaryPredicate __binary_pred, - input_iterator_tag, output_iterator_tag) + __unique_copy_1(_InputIterator __first, _InputIterator __last, + _OutputIterator __result, _BinaryPredicate __binary_pred, + __false_type) { - typename iterator_traits<_InputIterator>::value_type __value = *__first; - __decltype(__gnu_cxx::__ops::__iter_comp_val(__binary_pred)) - __rebound_pred - = __gnu_cxx::__ops::__iter_comp_val(__binary_pred); + typedef typename iterator_traits<_InputIterator>::value_type _Val; + _Val __value = *__first; *__result = __value; while (++__first != __last) - if (!__rebound_pred(__first, __value)) + if (!__binary_pred(std::__addressof(__value), __first)) { __value = *__first; *++__result = __value; @@ -971,19 +964,14 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION return ++__result; } - /** - * This is an uglified - * unique_copy(_InputIterator, _InputIterator, _OutputIterator, - * _BinaryPredicate) - * overloaded for input iterators and forward iterator as result. - */ + // Implementation of std::unique_copy for non-forward iterators, + // where we can compare with the last element written to the output. template<typename _InputIterator, typename _ForwardIterator, typename _BinaryPredicate> - _GLIBCXX20_CONSTEXPR _ForwardIterator - __unique_copy(_InputIterator __first, _InputIterator __last, - _ForwardIterator __result, _BinaryPredicate __binary_pred, - input_iterator_tag, forward_iterator_tag) + __unique_copy_1(_InputIterator __first, _InputIterator __last, + _ForwardIterator __result, _BinaryPredicate __binary_pred, + __true_type) { *__result = *__first; while (++__first != __last) @@ -992,6 +980,31 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION return ++__result; } + // Implementation of std::unique_copy for non-forward iterators. + // We cannot compare *i to *(i-1) so we need to either make a copy + // or compare with the last element written to the output range. + template<typename _InputIterator, typename _OutputIterator, + typename _BinaryPredicate> + _GLIBCXX20_CONSTEXPR + _OutputIterator + __unique_copy(_InputIterator __first, _InputIterator __last, + _OutputIterator __result, _BinaryPredicate __binary_pred, + input_iterator_tag) + { + // _GLIBCXX_RESOLVE_LIB_DEFECTS + // 2439. unique_copy() sometimes can't fall back to reading its output + typedef iterator_traits<_InputIterator> _InItTraits; + typedef iterator_traits<_OutputIterator> _OutItTraits; + typedef typename _OutItTraits::iterator_category _Cat; + const bool __output_is_fwd = __is_base_of(forward_iterator_tag, _Cat); + const bool __same_type = __is_same(typename _OutItTraits::value_type, + typename _InItTraits::value_type); + typedef __truth_type<__output_is_fwd && __same_type> __cmp_with_output; + return std::__unique_copy_1(__first, __last, __result, __binary_pred, + typename __cmp_with_output::__type()); + } + + /** * This is an uglified reverse(_BidirectionalIterator, * _BidirectionalIterator) @@ -4456,8 +4469,7 @@ _GLIBCXX_BEGIN_NAMESPACE_ALGO return __result; return std::__unique_copy(__first, __last, __result, __gnu_cxx::__ops::__iter_equal_to_iter(), - std::__iterator_category(__first), - std::__iterator_category(__result)); + std::__iterator_category(__first)); } /** @@ -4499,8 +4511,7 @@ _GLIBCXX_BEGIN_NAMESPACE_ALGO return __result; return std::__unique_copy(__first, __last, __result, __gnu_cxx::__ops::__iter_comp_iter(__binary_pred), - std::__iterator_category(__first), - std::__iterator_category(__result)); + std::__iterator_category(__first)); } #if __cplusplus <= 201103L || _GLIBCXX_USE_DEPRECATED diff --git a/libstdc++-v3/include/bits/stl_deque.h b/libstdc++-v3/include/bits/stl_deque.h index 8d8ee57..7055641 100644 --- a/libstdc++-v3/include/bits/stl_deque.h +++ b/libstdc++-v3/include/bits/stl_deque.h @@ -1331,7 +1331,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER { size_type __sz = this->_M_impl._M_finish - this->_M_impl._M_start; if (__sz > max_size ()) - __builtin_unreachable (); + __builtin_unreachable(); return __sz; } diff --git a/libstdc++-v3/include/bits/stl_vector.h b/libstdc++-v3/include/bits/stl_vector.h index 625c1c9..f2c1bce 100644 --- a/libstdc++-v3/include/bits/stl_vector.h +++ b/libstdc++-v3/include/bits/stl_vector.h @@ -372,8 +372,10 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER _GLIBCXX20_CONSTEXPR ~_Vector_base() _GLIBCXX_NOEXCEPT { - _M_deallocate(_M_impl._M_start, - _M_impl._M_end_of_storage - _M_impl._M_start); + ptrdiff_t __n = _M_impl._M_end_of_storage - _M_impl._M_start; + if (__n < 0) + __builtin_unreachable(); + _M_deallocate(_M_impl._M_start, size_t(__n)); } public: @@ -1106,7 +1108,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER { ptrdiff_t __dif = this->_M_impl._M_finish - this->_M_impl._M_start; if (__dif < 0) - __builtin_unreachable (); + __builtin_unreachable(); return size_type(__dif); } @@ -1198,7 +1200,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER ptrdiff_t __dif = this->_M_impl._M_end_of_storage - this->_M_impl._M_start; if (__dif < 0) - __builtin_unreachable (); + __builtin_unreachable(); return size_type(__dif); } diff --git a/libstdc++-v3/include/bits/vector.tcc b/libstdc++-v3/include/bits/vector.tcc index e18f01a..70ead1d 100644 --- a/libstdc++-v3/include/bits/vector.tcc +++ b/libstdc++-v3/include/bits/vector.tcc @@ -466,7 +466,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER { const size_type __len = _M_check_len(1u, "vector::_M_realloc_insert"); if (__len <= 0) - __builtin_unreachable (); + __builtin_unreachable(); pointer __old_start = this->_M_impl._M_start; pointer __old_finish = this->_M_impl._M_finish; const size_type __elems_before = __position - begin(); @@ -573,10 +573,10 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER { const size_type __len = _M_check_len(1u, "vector::_M_realloc_append"); if (__len <= 0) - __builtin_unreachable (); + __builtin_unreachable(); pointer __old_start = this->_M_impl._M_start; pointer __old_finish = this->_M_impl._M_finish; - const size_type __elems = end() - begin(); + const size_type __elems = size(); pointer __new_start(this->_M_allocate(__len)); pointer __new_finish(__new_start); diff --git a/libstdc++-v3/include/bits/version.def b/libstdc++-v3/include/bits/version.def index 5a981ca..5efe4d1 100644 --- a/libstdc++-v3/include/bits/version.def +++ b/libstdc++-v3/include/bits/version.def @@ -1363,7 +1363,7 @@ ftms = { v = 201907; cxxmin = 20; hosted = yes; - extra_cond = "__glibcxx_atomic_wait || _GLIBCXX_HAVE_POSIX_SEMAPHORE"; + extra_cond = "__glibcxx_atomic_wait"; }; }; @@ -1978,6 +1978,15 @@ ftms = { }; }; +ftms = { + name = polymorphic; + values = { + v = 202502; + cxxmin = 26; + hosted = yes; + }; +}; + // Standard test specifications. stds[97] = ">= 199711L"; stds[03] = ">= 199711L"; diff --git a/libstdc++-v3/include/bits/version.h b/libstdc++-v3/include/bits/version.h index 0664611..5e905da 100644 --- a/libstdc++-v3/include/bits/version.h +++ b/libstdc++-v3/include/bits/version.h @@ -1499,7 +1499,7 @@ #undef __glibcxx_want_move_iterator_concept #if !defined(__cpp_lib_semaphore) -# if (__cplusplus >= 202002L) && _GLIBCXX_HOSTED && (__glibcxx_atomic_wait || _GLIBCXX_HAVE_POSIX_SEMAPHORE) +# if (__cplusplus >= 202002L) && _GLIBCXX_HOSTED && (__glibcxx_atomic_wait) # define __glibcxx_semaphore 201907L # if defined(__glibcxx_want_all) || defined(__glibcxx_want_semaphore) # define __cpp_lib_semaphore 201907L @@ -2213,4 +2213,14 @@ #endif /* !defined(__cpp_lib_indirect) && defined(__glibcxx_want_indirect) */ #undef __glibcxx_want_indirect +#if !defined(__cpp_lib_polymorphic) +# if (__cplusplus > 202302L) && _GLIBCXX_HOSTED +# define __glibcxx_polymorphic 202502L +# if defined(__glibcxx_want_all) || defined(__glibcxx_want_polymorphic) +# define __cpp_lib_polymorphic 202502L +# endif +# endif +#endif /* !defined(__cpp_lib_polymorphic) && defined(__glibcxx_want_polymorphic) */ +#undef __glibcxx_want_polymorphic + #undef __glibcxx_want_all diff --git a/libstdc++-v3/include/std/barrier b/libstdc++-v3/include/std/barrier index 6c3cfd4..56270c9 100644 --- a/libstdc++-v3/include/std/barrier +++ b/libstdc++-v3/include/std/barrier @@ -81,105 +81,142 @@ It looks different from literature pseudocode for two main reasons: enum class __barrier_phase_t : unsigned char { }; - template<typename _CompletionF> - class __tree_barrier + struct __tree_barrier_base + { + static constexpr ptrdiff_t + max() noexcept + { return __PTRDIFF_MAX__ - 1; } + + protected: + using __atomic_phase_ref_t = std::__atomic_ref<__barrier_phase_t>; + using __atomic_phase_const_ref_t = std::__atomic_ref<const __barrier_phase_t>; + static constexpr auto __phase_alignment = + __atomic_phase_ref_t::required_alignment; + + using __tickets_t = std::array<__barrier_phase_t, 64>; + struct alignas(64) /* naturally-align the heap state */ __state_t { - using __atomic_phase_ref_t = std::__atomic_ref<__barrier_phase_t>; - using __atomic_phase_const_ref_t = std::__atomic_ref<const __barrier_phase_t>; - static constexpr auto __phase_alignment = - __atomic_phase_ref_t::required_alignment; + alignas(__phase_alignment) __tickets_t __tickets; + }; - using __tickets_t = std::array<__barrier_phase_t, 64>; - struct alignas(64) /* naturally-align the heap state */ __state_t - { - alignas(__phase_alignment) __tickets_t __tickets; - }; + ptrdiff_t _M_expected; + __atomic_base<__state_t*> _M_state{nullptr}; + __atomic_base<ptrdiff_t> _M_expected_adjustment{0}; + alignas(__phase_alignment) __barrier_phase_t _M_phase{}; - ptrdiff_t _M_expected; - unique_ptr<__state_t[]> _M_state; - __atomic_base<ptrdiff_t> _M_expected_adjustment; - _CompletionF _M_completion; + explicit constexpr + __tree_barrier_base(ptrdiff_t __expected) + : _M_expected(__expected) + { + __glibcxx_assert(__expected >= 0 && __expected <= max()); - alignas(__phase_alignment) __barrier_phase_t _M_phase; + if (!std::is_constant_evaluated()) + _M_state.store(_M_alloc_state().release(), memory_order_release); + } - bool - _M_arrive(__barrier_phase_t __old_phase, size_t __current) - { - const auto __old_phase_val = static_cast<unsigned char>(__old_phase); - const auto __half_step = - static_cast<__barrier_phase_t>(__old_phase_val + 1); - const auto __full_step = - static_cast<__barrier_phase_t>(__old_phase_val + 2); + unique_ptr<__state_t[]> + _M_alloc_state() + { + size_t const __count = (_M_expected + 1) >> 1; + return std::make_unique<__state_t[]>(__count); + } - size_t __current_expected = _M_expected; - __current %= ((_M_expected + 1) >> 1); + bool + _M_arrive(__barrier_phase_t __old_phase, size_t __current) + { + const auto __old_phase_val = static_cast<unsigned char>(__old_phase); + const auto __half_step = + static_cast<__barrier_phase_t>(__old_phase_val + 1); + const auto __full_step = + static_cast<__barrier_phase_t>(__old_phase_val + 2); + + size_t __current_expected = _M_expected; + __current %= ((_M_expected + 1) >> 1); + + __state_t* const __state = _M_state.load(memory_order_relaxed); + + for (int __round = 0; ; ++__round) + { + if (__current_expected <= 1) + return true; + size_t const __end_node = ((__current_expected + 1) >> 1), + __last_node = __end_node - 1; + for ( ; ; ++__current) + { + if (__current == __end_node) + __current = 0; + auto __expect = __old_phase; + __atomic_phase_ref_t __phase(__state[__current] + .__tickets[__round]); + if (__current == __last_node && (__current_expected & 1)) + { + if (__phase.compare_exchange_strong(__expect, __full_step, + memory_order_acq_rel)) + break; // I'm 1 in 1, go to next __round + } + else if (__phase.compare_exchange_strong(__expect, __half_step, + memory_order_acq_rel)) + { + return false; // I'm 1 in 2, done with arrival + } + else if (__expect == __half_step) + { + if (__phase.compare_exchange_strong(__expect, __full_step, + memory_order_acq_rel)) + break; // I'm 2 in 2, go to next __round + } + } + __current_expected = __last_node + 1; + __current >>= 1; + } + } + }; - for (int __round = 0; ; ++__round) - { - if (__current_expected <= 1) - return true; - size_t const __end_node = ((__current_expected + 1) >> 1), - __last_node = __end_node - 1; - for ( ; ; ++__current) - { - if (__current == __end_node) - __current = 0; - auto __expect = __old_phase; - __atomic_phase_ref_t __phase(_M_state[__current] - .__tickets[__round]); - if (__current == __last_node && (__current_expected & 1)) - { - if (__phase.compare_exchange_strong(__expect, __full_step, - memory_order_acq_rel)) - break; // I'm 1 in 1, go to next __round - } - else if (__phase.compare_exchange_strong(__expect, __half_step, - memory_order_acq_rel)) - { - return false; // I'm 1 in 2, done with arrival - } - else if (__expect == __half_step) - { - if (__phase.compare_exchange_strong(__expect, __full_step, - memory_order_acq_rel)) - break; // I'm 2 in 2, go to next __round - } - } - __current_expected = __last_node + 1; - __current >>= 1; - } - } + template<typename _CompletionF> + class __tree_barrier : public __tree_barrier_base + { + [[no_unique_address]] _CompletionF _M_completion; + + // _GLIBCXX_RESOLVE_LIB_DEFECTS + // 3898. Possibly unintended preconditions for completion functions + void _M_invoke_completion() noexcept { _M_completion(); } public: using arrival_token = __barrier_phase_t; - static constexpr ptrdiff_t - max() noexcept - { return __PTRDIFF_MAX__; } - + constexpr __tree_barrier(ptrdiff_t __expected, _CompletionF __completion) - : _M_expected(__expected), _M_expected_adjustment(0), - _M_completion(move(__completion)), - _M_phase(static_cast<__barrier_phase_t>(0)) - { - size_t const __count = (_M_expected + 1) >> 1; - - _M_state = std::make_unique<__state_t[]>(__count); - } + : __tree_barrier_base(__expected), _M_completion(std::move(__completion)) + { } [[nodiscard]] arrival_token arrive(ptrdiff_t __update) { + __glibcxx_assert(__update > 0); + // FIXME: Check that update is less than or equal to the expected count + // for the current barrier phase. + std::hash<std::thread::id> __hasher; size_t __current = __hasher(std::this_thread::get_id()); __atomic_phase_ref_t __phase(_M_phase); const auto __old_phase = __phase.load(memory_order_relaxed); const auto __cur = static_cast<unsigned char>(__old_phase); - for(; __update; --__update) + + if (__cur == 0 && !_M_state.load(memory_order_relaxed)) [[unlikely]] + { + auto __p = _M_alloc_state(); + __state_t* __val = nullptr; + if (_M_state.compare_exchange_strong(__val, __p.get(), + memory_order_seq_cst, + memory_order_acquire)) + __p.release(); + } + + for (; __update; --__update) { - if(_M_arrive(__old_phase, __current)) + if (_M_arrive(__old_phase, __current)) { - _M_completion(); + _M_invoke_completion(); _M_expected += _M_expected_adjustment.load(memory_order_relaxed); _M_expected_adjustment.store(0, memory_order_relaxed); auto __new_phase = static_cast<__barrier_phase_t>(__cur + 2); @@ -194,11 +231,7 @@ It looks different from literature pseudocode for two main reasons: wait(arrival_token&& __old_phase) const { __atomic_phase_const_ref_t __phase(_M_phase); - auto const __test_fn = [=] - { - return __phase.load(memory_order_acquire) != __old_phase; - }; - std::__atomic_wait_address(&_M_phase, __test_fn); + __phase.wait(__old_phase, memory_order_acquire); } void @@ -212,6 +245,8 @@ It looks different from literature pseudocode for two main reasons: template<typename _CompletionF = __empty_completion> class barrier { + static_assert(is_invocable_v<_CompletionF&>); + // Note, we may introduce a "central" barrier algorithm at some point // for more space constrained targets using __algorithm_t = __tree_barrier<_CompletionF>; @@ -236,7 +271,7 @@ It looks different from literature pseudocode for two main reasons: max() noexcept { return __algorithm_t::max(); } - explicit + constexpr explicit barrier(ptrdiff_t __count, _CompletionF __completion = _CompletionF()) : _M_b(__count, std::move(__completion)) { } diff --git a/libstdc++-v3/include/std/bit b/libstdc++-v3/include/std/bit index 5187c96..fd75edf 100644 --- a/libstdc++-v3/include/std/bit +++ b/libstdc++-v3/include/std/bit @@ -166,7 +166,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION // Variant for power of two _Nd which the compiler can // easily pattern match. constexpr unsigned __uNd = _Nd; - const unsigned __r = __s; + const auto __r = static_cast<unsigned>(__s); return (__x << (__r % __uNd)) | (__x >> ((-__r) % __uNd)); } const int __r = __s % _Nd; @@ -188,7 +188,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION // Variant for power of two _Nd which the compiler can // easily pattern match. constexpr unsigned __uNd = _Nd; - const unsigned __r = __s; + const auto __r = static_cast<unsigned>(__s); return (__x >> (__r % __uNd)) | (__x << ((-__r) % __uNd)); } const int __r = __s % _Nd; diff --git a/libstdc++-v3/include/std/flat_map b/libstdc++-v3/include/std/flat_map index 6593988..4bd4963 100644 --- a/libstdc++-v3/include/std/flat_map +++ b/libstdc++-v3/include/std/flat_map @@ -873,7 +873,10 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION [[nodiscard]] friend bool operator==(const _Derived& __x, const _Derived& __y) - { return std::equal(__x.begin(), __x.end(), __y.begin(), __y.end()); } + { + return __x._M_cont.keys == __y._M_cont.keys + && __x._M_cont.values == __y._M_cont.values; + } template<typename _Up = value_type> [[nodiscard]] @@ -895,7 +898,10 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION { auto __guard = _M_make_clear_guard(); auto __zv = views::zip(_M_cont.keys, _M_cont.values); - auto __sr = ranges::remove_if(__zv, __pred); + auto __sr = ranges::remove_if(__zv, __pred, + [](const auto& __e) { + return const_reference(__e); + }); auto __erased = __sr.size(); erase(end() - __erased, end()); __guard._M_disable(); diff --git a/libstdc++-v3/include/std/latch b/libstdc++-v3/include/std/latch index dc147c2..9504df0 100644 --- a/libstdc++-v3/include/std/latch +++ b/libstdc++-v3/include/std/latch @@ -89,15 +89,33 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION _GLIBCXX_ALWAYS_INLINE void wait() const noexcept { - auto const __pred = [this] { return this->try_wait(); }; - std::__atomic_wait_address(&_M_counter, __pred); + auto const __vfn = [this] { + return __atomic_impl::load(&_M_counter, memory_order::acquire); + }; + auto const __pred = [](__detail::__platform_wait_t __v) { + return __v == 0; + }; + std::__atomic_wait_address(&_M_counter, __pred, __vfn); } _GLIBCXX_ALWAYS_INLINE void arrive_and_wait(ptrdiff_t __update = 1) noexcept { - count_down(__update); - wait(); + // The standard specifies this functions as count_down(update); wait(); + // but we combine those two calls into one and avoid the wait() if we + // know the counter reached zero. + + __glibcxx_assert(__update >= 0 && __update <= max()); + // Use acq_rel here because an omitted wait() would have used acquire: + auto const __old = __atomic_impl::fetch_sub(&_M_counter, __update, + memory_order::acq_rel); + if (std::cmp_equal(__old, __update)) + __atomic_impl::notify_all(&_M_counter); + else + { + __glibcxx_assert(std::cmp_less(__update, __old)); + wait(); + } } private: diff --git a/libstdc++-v3/include/std/memory b/libstdc++-v3/include/std/memory index d64e65c..1da03b3 100644 --- a/libstdc++-v3/include/std/memory +++ b/libstdc++-v3/include/std/memory @@ -113,6 +113,7 @@ #define __glibcxx_want_make_unique #define __glibcxx_want_out_ptr #define __glibcxx_want_parallel_algorithm +#define __glibcxx_want_polymorphic #define __glibcxx_want_ranges #define __glibcxx_want_raw_memory_algorithms #define __glibcxx_want_shared_ptr_arrays diff --git a/libstdc++-v3/include/std/semaphore b/libstdc++-v3/include/std/semaphore index bec5ac3..ca1bffe 100644 --- a/libstdc++-v3/include/std/semaphore +++ b/libstdc++-v3/include/std/semaphore @@ -35,29 +35,29 @@ #include <bits/requires_hosted.h> // concurrency -#if __cplusplus > 201703L -#include <bits/semaphore_base.h> - #define __glibcxx_want_semaphore #include <bits/version.h> -#ifdef __cpp_lib_semaphore // C++ >= 20 && hosted && (atomic_wait || posix_sem) +#ifdef __cpp_lib_semaphore // C++ >= 20 && hosted && atomic_wait +#include <bits/semaphore_base.h> + namespace std _GLIBCXX_VISIBILITY(default) { _GLIBCXX_BEGIN_NAMESPACE_VERSION - template<ptrdiff_t __least_max_value = __semaphore_impl::_S_max> + template<ptrdiff_t __least_max_value = __semaphore_base<true>::_S_max> class counting_semaphore { static_assert(__least_max_value >= 0); - static_assert(__least_max_value <= __semaphore_impl::_S_max); - __semaphore_impl _M_sem; + using _Impl = __semaphore_impl<__least_max_value>; + _Impl _M_sem; public: - explicit counting_semaphore(ptrdiff_t __desired) noexcept - : _M_sem(__desired) - { } + constexpr explicit + counting_semaphore(ptrdiff_t __desired) noexcept + : _M_sem(__desired) + { __glibcxx_assert(__desired >= 0 && __desired <= max()); } ~counting_semaphore() = default; @@ -69,8 +69,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION { return __least_max_value; } void - release(ptrdiff_t __update = 1) noexcept(noexcept(_M_sem._M_release(1))) - { _M_sem._M_release(__update); } + release(ptrdiff_t __update = 1) noexcept + { + [[maybe_unused]] ptrdiff_t __old = _M_sem._M_release(__update); + __glibcxx_assert(__update >= 0 && __update <= max() - __old); + } void acquire() noexcept(noexcept(_M_sem._M_acquire())) @@ -91,10 +94,13 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION { return _M_sem._M_try_acquire_until(__atime); } }; + /** @brief A binary semaphore + * + * @since C++20 + */ using binary_semaphore = std::counting_semaphore<1>; _GLIBCXX_END_NAMESPACE_VERSION } // namespace -#endif // cpp_lib_atomic_wait || _GLIBCXX_HAVE_POSIX_SEMAPHORE #endif // __cpp_lib_semaphore #endif // _GLIBCXX_SEMAPHORE diff --git a/libstdc++-v3/include/std/stop_token b/libstdc++-v3/include/std/stop_token index 1225b3a..775ec6a 100644 --- a/libstdc++-v3/include/std/stop_token +++ b/libstdc++-v3/include/std/stop_token @@ -34,8 +34,7 @@ #define __glibcxx_want_jthread #include <bits/version.h> -#if __cplusplus > 201703L - +#ifdef __glibcxx_jthread // C++ >= 20 #include <atomic> #include <bits/std_thread.h> @@ -650,6 +649,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION stop_callback(stop_token, _Callback) -> stop_callback<_Callback>; _GLIBCXX_END_NAMESPACE_VERSION -} // namespace -#endif // __cplusplus > 201703L +} // namespace std +#endif // __glibcxx_jthread #endif // _GLIBCXX_STOP_TOKEN diff --git a/libstdc++-v3/include/std/type_traits b/libstdc++-v3/include/std/type_traits index 6bf355d..c8907fe 100644 --- a/libstdc++-v3/include/std/type_traits +++ b/libstdc++-v3/include/std/type_traits @@ -1039,6 +1039,13 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION // Destructible and constructible type properties. +#if _GLIBCXX_USE_BUILTIN_TRAIT(__is_destructible) + /// is_destructible + template<typename _Tp> + struct is_destructible + : public __bool_constant<__is_destructible(_Tp)> + { }; +#else // In N3290 is_destructible does not say anything about function // types and abstract types, see LWG 2049. This implementation // describes function types as non-destructible and all complete @@ -1090,7 +1097,15 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}), "template argument must be a complete class or an unbounded array"); }; +#endif +#if _GLIBCXX_USE_BUILTIN_TRAIT(__is_nothrow_destructible) + /// is_nothrow_destructible + template<typename _Tp> + struct is_nothrow_destructible + : public __bool_constant<__is_nothrow_destructible(_Tp)> + { }; +#else /// @cond undocumented // is_nothrow_destructible requires that is_destructible is @@ -1144,6 +1159,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}), "template argument must be a complete class or an unbounded array"); }; +#endif /// @cond undocumented template<typename _Tp, typename... _Args> @@ -1451,6 +1467,13 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION "template argument must be a complete class or an unbounded array"); }; +#if _GLIBCXX_USE_BUILTIN_TRAIT(__is_trivially_destructible) + /// is_trivially_destructible + template<typename _Tp> + struct is_trivially_destructible + : public __bool_constant<__is_trivially_destructible(_Tp)> + { }; +#else /// is_trivially_destructible template<typename _Tp> struct is_trivially_destructible @@ -1460,7 +1483,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}), "template argument must be a complete class or an unbounded array"); }; - +#endif /// has_virtual_destructor template<typename _Tp> @@ -3581,8 +3604,13 @@ template <typename _Tp> inline constexpr bool is_move_assignable_v = __is_assignable(__add_lval_ref_t<_Tp>, __add_rval_ref_t<_Tp>); +#if _GLIBCXX_USE_BUILTIN_TRAIT(__is_destructible) +template <typename _Tp> + inline constexpr bool is_destructible_v = __is_destructible(_Tp); +#else template <typename _Tp> inline constexpr bool is_destructible_v = is_destructible<_Tp>::value; +#endif template <typename _Tp, typename... _Args> inline constexpr bool is_trivially_constructible_v @@ -3609,7 +3637,11 @@ template <typename _Tp> = __is_trivially_assignable(__add_lval_ref_t<_Tp>, __add_rval_ref_t<_Tp>); -#if __cpp_concepts +#if _GLIBCXX_USE_BUILTIN_TRAIT(__is_trivially_destructible) +template <typename _Tp> + inline constexpr bool is_trivially_destructible_v + = __is_trivially_destructible(_Tp); +#elif __cpp_concepts template <typename _Tp> inline constexpr bool is_trivially_destructible_v = false; @@ -3654,9 +3686,15 @@ template <typename _Tp> inline constexpr bool is_nothrow_move_assignable_v = __is_nothrow_assignable(__add_lval_ref_t<_Tp>, __add_rval_ref_t<_Tp>); +#if _GLIBCXX_USE_BUILTIN_TRAIT(__is_nothrow_destructible) +template <typename _Tp> + inline constexpr bool is_nothrow_destructible_v + = __is_nothrow_destructible(_Tp); +#else template <typename _Tp> inline constexpr bool is_nothrow_destructible_v = is_nothrow_destructible<_Tp>::value; +#endif template <typename _Tp> inline constexpr bool has_virtual_destructor_v |