libstdc++: Add always_inline to most allocator functions

This reduces the abstraction penalty for allocator support in
unoptimized code. Constructing and using allocators in containers calls
many one-line (or completely empty) inline functions. Those can all be
inlined to reduce code size and function call overhead for -O0.

libstdc++-v3/ChangeLog:

	* include/bits/alloc_traits.h (allocator_traits): Add
	always_inline attribute to all member functions.
	(__do_alloc_on_copy, __alloc_on_copy, __do_alloc_on_move)
	(__alloc_on_move, __do_alloc_on_swap, __alloc_on_swap)
	(_Destroy(FwdIter, FwdIter, allocator<T>&)): : Add
	always_inline attribute.
	* include/bits/allocator.h (allocator): Add always_inline
	attribute to all member functions and equality operators.
	* include/bits/new_allocator.h (__new_allocator): Likewise.
	* include/ext/alloc_traits.h (__gnu_cxx::__alloc_traits):
	Likewise.

4 files changed, 72 insertions, 15 deletions

diff --git a/libstdc++-v3/include/bits/alloc_traits.h b/libstdc++-v3/include/bits/alloc_traits.h
index 8479bfd..203988a 100644
--- a/libstdc++-v3/include/bits/alloc_traits.h
+++ b/libstdc++-v3/include/bits/alloc_traits.h
@@ -463,7 +463,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *
        *  Calls @c a.allocate(n)
       */
-      _GLIBCXX_NODISCARD static _GLIBCXX20_CONSTEXPR pointer
+      [[__nodiscard__,__gnu__::__always_inline__]]
+      static _GLIBCXX20_CONSTEXPR pointer
       allocate(allocator_type& __a, size_type __n)
       { return __a.allocate(__n); }
 
@@ -477,7 +478,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *
        *  Returns <tt> a.allocate(n, hint) </tt>
       */
-      _GLIBCXX_NODISCARD static _GLIBCXX20_CONSTEXPR pointer
+      [[__nodiscard__,__gnu__::__always_inline__]]
+      static _GLIBCXX20_CONSTEXPR pointer
       allocate(allocator_type& __a, size_type __n, const_void_pointer __hint)
       {
 #if __cplusplus <= 201703L
@@ -495,6 +497,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *
        *  Calls <tt> a.deallocate(p, n) </tt>
       */
+      [[__gnu__::__always_inline__]]
       static _GLIBCXX20_CONSTEXPR void
       deallocate(allocator_type& __a, pointer __p, size_type __n)
       { __a.deallocate(__p, __n); }
@@ -511,6 +514,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  `std::construct_at(__p, std::forward<_Args>(__args)...)` instead.
       */
       template<typename _Up, typename... _Args>
+	[[__gnu__::__always_inline__]]
 	static _GLIBCXX20_CONSTEXPR void
 	construct(allocator_type& __a __attribute__((__unused__)), _Up* __p,
 		  _Args&&... __args)
@@ -531,6 +535,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  Calls @c __a.destroy(__p).
       */
       template<typename _Up>
+	[[__gnu__::__always_inline__]]
 	static _GLIBCXX20_CONSTEXPR void
 	destroy(allocator_type& __a __attribute__((__unused__)), _Up* __p)
 	noexcept(is_nothrow_destructible<_Up>::value)
@@ -547,6 +552,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  @param  __a  An allocator.
        *  @return @c __a.max_size()
       */
+      [[__gnu__::__always_inline__]]
       static _GLIBCXX20_CONSTEXPR size_type
       max_size(const allocator_type& __a __attribute__((__unused__))) noexcept
       {
@@ -562,6 +568,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  @param  __rhs  An allocator.
        *  @return @c __rhs
       */
+      [[__gnu__::__always_inline__]]
       static _GLIBCXX20_CONSTEXPR allocator_type
       select_on_container_copy_construction(const allocator_type& __rhs)
       { return __rhs; }
@@ -633,6 +640,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  `std::construct_at(__p, std::forward<_Args>(__args)...)` instead.
       */
       template<typename _Up, typename... _Args>
+	[[__gnu__::__always_inline__]]
 	static _GLIBCXX20_CONSTEXPR void
 	construct(allocator_type&, _Up* __p, _Args&&... __args)
 	noexcept(std::is_nothrow_constructible<_Up, _Args...>::value)
@@ -646,6 +654,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  Invokes the destructor for `*__p`.
       */
       template<typename _Up>
+	[[__gnu__::__always_inline__]]
 	static _GLIBCXX20_CONSTEXPR void
 	destroy(allocator_type&, _Up* __p)
 	noexcept(is_nothrow_destructible<_Up>::value)
@@ -660,6 +669,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  @param  __rhs  An allocator.
        *  @return `__rhs`
       */
+      [[__gnu__::__always_inline__]]
       static _GLIBCXX20_CONSTEXPR allocator_type
       select_on_container_copy_construction(const allocator_type& __rhs)
       { return __rhs; }
@@ -669,22 +679,26 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   /// @cond undocumented
 #if __cplusplus < 201703L
   template<typename _Alloc>
+    [[__gnu__::__always_inline__]]
     inline void
     __do_alloc_on_copy(_Alloc& __one, const _Alloc& __two, true_type)
     { __one = __two; }
 
   template<typename _Alloc>
+    [[__gnu__::__always_inline__]]
     inline void
     __do_alloc_on_copy(_Alloc&, const _Alloc&, false_type)
     { }
 #endif
 
   template<typename _Alloc>
+    [[__gnu__::__always_inline__]]
     _GLIBCXX14_CONSTEXPR inline void
     __alloc_on_copy(_Alloc& __one, const _Alloc& __two)
     {
-      typedef allocator_traits<_Alloc> __traits;
-      typedef typename __traits::propagate_on_container_copy_assignment __pocca;
+      using __traits = allocator_traits<_Alloc>;
+      using __pocca =
+	typename __traits::propagate_on_container_copy_assignment::type;
 #if __cplusplus >= 201703L
       if constexpr (__pocca::value)
 	__one = __two;
@@ -694,6 +708,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     }
 
   template<typename _Alloc>
+    [[__gnu__::__always_inline__]]
     constexpr _Alloc
     __alloc_on_copy(const _Alloc& __a)
     {
@@ -703,20 +718,24 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
 #if __cplusplus < 201703L
   template<typename _Alloc>
+    [[__gnu__::__always_inline__]]
     inline void __do_alloc_on_move(_Alloc& __one, _Alloc& __two, true_type)
     { __one = std::move(__two); }
 
   template<typename _Alloc>
+    [[__gnu__::__always_inline__]]
     inline void __do_alloc_on_move(_Alloc&, _Alloc&, false_type)
     { }
 #endif
 
   template<typename _Alloc>
+    [[__gnu__::__always_inline__]]
     _GLIBCXX14_CONSTEXPR inline void
     __alloc_on_move(_Alloc& __one, _Alloc& __two)
     {
-      typedef allocator_traits<_Alloc> __traits;
-      typedef typename __traits::propagate_on_container_move_assignment __pocma;
+      using __traits = allocator_traits<_Alloc>;
+      using __pocma
+	= typename __traits::propagate_on_container_move_assignment::type;
 #if __cplusplus >= 201703L
       if constexpr (__pocma::value)
 	__one = std::move(__two);
@@ -727,6 +746,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
 #if __cplusplus < 201703L
   template<typename _Alloc>
+    [[__gnu__::__always_inline__]]
     inline void __do_alloc_on_swap(_Alloc& __one, _Alloc& __two, true_type)
     {
       using std::swap;
@@ -734,16 +754,18 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     }
 
   template<typename _Alloc>
+    [[__gnu__::__always_inline__]]
     inline void __do_alloc_on_swap(_Alloc&, _Alloc&, false_type)
     { }
 #endif
 
   template<typename _Alloc>
+    [[__gnu__::__always_inline__]]
     _GLIBCXX14_CONSTEXPR inline void
     __alloc_on_swap(_Alloc& __one, _Alloc& __two)
     {
-      typedef allocator_traits<_Alloc> __traits;
-      typedef typename __traits::propagate_on_container_swap __pocs;
+      using __traits = allocator_traits<_Alloc>;
+      using __pocs = typename __traits::propagate_on_container_swap::type;
 #if __cplusplus >= 201703L
       if constexpr (__pocs::value)
 	{
@@ -904,7 +926,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
 #if _GLIBCXX_HOSTED
   template<typename _ForwardIterator, typename _Tp>
-    _GLIBCXX20_CONSTEXPR
+    __attribute__((__always_inline__)) _GLIBCXX20_CONSTEXPR
     inline void
     _Destroy(_ForwardIterator __first, _ForwardIterator __last,
 	     allocator<_Tp>&)
diff --git a/libstdc++-v3/include/bits/allocator.h b/libstdc++-v3/include/bits/allocator.h
index 54f5acf..6763344 100644
--- a/libstdc++-v3/include/bits/allocator.h
+++ b/libstdc++-v3/include/bits/allocator.h
@@ -105,6 +105,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       ~allocator() = default;
 
       template<typename _Up>
+	__attribute__((__always_inline__))
 	constexpr
 	allocator(const allocator<_Up>&) noexcept { }
 
@@ -157,9 +158,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
       // _GLIBCXX_RESOLVE_LIB_DEFECTS
       // 3035. std::allocator's constructors should be constexpr
+      __attribute__((__always_inline__))
       _GLIBCXX20_CONSTEXPR
       allocator() _GLIBCXX_NOTHROW { }
 
+      __attribute__((__always_inline__))
       _GLIBCXX20_CONSTEXPR
       allocator(const allocator& __a) _GLIBCXX_NOTHROW
       : __allocator_base<_Tp>(__a) { }
@@ -170,9 +173,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #endif
 
       template<typename _Tp1>
+	__attribute__((__always_inline__))
 	_GLIBCXX20_CONSTEXPR
 	allocator(const allocator<_Tp1>&) _GLIBCXX_NOTHROW { }
 
+      __attribute__((__always_inline__))
 #if __cpp_constexpr_dynamic_alloc
       constexpr
 #endif
@@ -206,12 +211,14 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       }
 #endif // C++20
 
-      friend _GLIBCXX20_CONSTEXPR bool
+      friend __attribute__((__always_inline__)) _GLIBCXX20_CONSTEXPR
+      bool
       operator==(const allocator&, const allocator&) _GLIBCXX_NOTHROW
       { return true; }
 
 #if __cpp_impl_three_way_comparison < 201907L
-      friend _GLIBCXX20_CONSTEXPR bool
+      friend __attribute__((__always_inline__)) _GLIBCXX20_CONSTEXPR
+      bool
       operator!=(const allocator&, const allocator&) _GLIBCXX_NOTHROW
       { return false; }
 #endif
@@ -225,6 +232,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
    * @relates std::allocator
    */
   template<typename _T1, typename _T2>
+    __attribute__((__always_inline__))
     inline _GLIBCXX20_CONSTEXPR bool
     operator==(const allocator<_T1>&, const allocator<_T2>&)
     _GLIBCXX_NOTHROW
@@ -232,6 +240,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
 #if __cpp_impl_three_way_comparison < 201907L
   template<typename _T1, typename _T2>
+    __attribute__((__always_inline__))
     inline _GLIBCXX20_CONSTEXPR bool
     operator!=(const allocator<_T1>&, const allocator<_T2>&)
     _GLIBCXX_NOTHROW
diff --git a/libstdc++-v3/include/bits/new_allocator.h b/libstdc++-v3/include/bits/new_allocator.h
index 92ae984..b954acd 100644
--- a/libstdc++-v3/include/bits/new_allocator.h
+++ b/libstdc++-v3/include/bits/new_allocator.h
@@ -83,13 +83,16 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       typedef std::true_type propagate_on_container_move_assignment;
 #endif
 
+      __attribute__((__always_inline__))
       _GLIBCXX20_CONSTEXPR
       __new_allocator() _GLIBCXX_USE_NOEXCEPT { }
 
+      __attribute__((__always_inline__))
       _GLIBCXX20_CONSTEXPR
       __new_allocator(const __new_allocator&) _GLIBCXX_USE_NOEXCEPT { }
 
       template<typename _Tp1>
+	__attribute__((__always_inline__))
 	_GLIBCXX20_CONSTEXPR
 	__new_allocator(const __new_allocator<_Tp1>&) _GLIBCXX_USE_NOEXCEPT { }
 
@@ -170,18 +173,21 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #undef _GLIBCXX_OPERATOR_NEW
 
 #if __cplusplus <= 201703L
+      __attribute__((__always_inline__))
       size_type
       max_size() const _GLIBCXX_USE_NOEXCEPT
       { return _M_max_size(); }
 
 #if __cplusplus >= 201103L
       template<typename _Up, typename... _Args>
+	__attribute__((__always_inline__))
 	void
 	construct(_Up* __p, _Args&&... __args)
 	noexcept(std::is_nothrow_constructible<_Up, _Args...>::value)
 	{ ::new((void *)__p) _Up(std::forward<_Args>(__args)...); }
 
       template<typename _Up>
+	__attribute__((__always_inline__))
 	void
 	destroy(_Up* __p)
 	noexcept(std::is_nothrow_destructible<_Up>::value)
@@ -189,30 +195,33 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #else
       // _GLIBCXX_RESOLVE_LIB_DEFECTS
       // 402. wrong new expression in [some_] allocator::construct
+      __attribute__((__always_inline__))
       void
       construct(pointer __p, const _Tp& __val)
       { ::new((void *)__p) _Tp(__val); }
 
+      __attribute__((__always_inline__))
       void
       destroy(pointer __p) { __p->~_Tp(); }
 #endif
 #endif // ! C++20
 
       template<typename _Up>
-	friend _GLIBCXX20_CONSTEXPR bool
+	friend __attribute__((__always_inline__)) _GLIBCXX20_CONSTEXPR bool
 	operator==(const __new_allocator&, const __new_allocator<_Up>&)
 	_GLIBCXX_NOTHROW
 	{ return true; }
 
 #if __cpp_impl_three_way_comparison < 201907L
       template<typename _Up>
-	friend _GLIBCXX20_CONSTEXPR bool
+	friend __attribute__((__always_inline__)) _GLIBCXX20_CONSTEXPR bool
 	operator!=(const __new_allocator&, const __new_allocator<_Up>&)
 	_GLIBCXX_NOTHROW
 	{ return false; }
 #endif
 
     private:
+      __attribute__((__always_inline__))
       _GLIBCXX_CONSTEXPR size_type
       _M_max_size() const _GLIBCXX_USE_NOEXCEPT
       {
diff --git a/libstdc++-v3/include/ext/alloc_traits.h b/libstdc++-v3/include/ext/alloc_traits.h
index c9547c7..d7008ee 100644
--- a/libstdc++-v3/include/ext/alloc_traits.h
+++ b/libstdc++-v3/include/ext/alloc_traits.h
@@ -73,6 +73,7 @@ template<typename _Alloc, typename = typename _Alloc::value_type>
   public:
     // overload construct for non-standard pointer types
     template<typename _Ptr, typename... _Args>
+      [[__gnu__::__always_inline__]]
       static _GLIBCXX14_CONSTEXPR
       std::__enable_if_t<__is_custom_pointer<_Ptr>::value>
       construct(_Alloc& __a, _Ptr __p, _Args&&... __args)
@@ -85,30 +86,38 @@ template<typename _Alloc, typename = typename _Alloc::value_type>
 
     // overload destroy for non-standard pointer types
     template<typename _Ptr>
+      [[__gnu__::__always_inline__]]
       static _GLIBCXX14_CONSTEXPR
       std::__enable_if_t<__is_custom_pointer<_Ptr>::value>
       destroy(_Alloc& __a, _Ptr __p)
       noexcept(noexcept(_Base_type::destroy(__a, std::__to_address(__p))))
       { _Base_type::destroy(__a, std::__to_address(__p)); }
 
+    [[__gnu__::__always_inline__]]
     static constexpr _Alloc _S_select_on_copy(const _Alloc& __a)
     { return _Base_type::select_on_container_copy_construction(__a); }
 
+    [[__gnu__::__always_inline__]]
     static _GLIBCXX14_CONSTEXPR void _S_on_swap(_Alloc& __a, _Alloc& __b)
     { std::__alloc_on_swap(__a, __b); }
 
+    [[__gnu__::__always_inline__]]
     static constexpr bool _S_propagate_on_copy_assign()
     { return _Base_type::propagate_on_container_copy_assignment::value; }
 
+    [[__gnu__::__always_inline__]]
     static constexpr bool _S_propagate_on_move_assign()
     { return _Base_type::propagate_on_container_move_assignment::value; }
 
+    [[__gnu__::__always_inline__]]
     static constexpr bool _S_propagate_on_swap()
     { return _Base_type::propagate_on_container_swap::value; }
 
+    [[__gnu__::__always_inline__]]
     static constexpr bool _S_always_equal()
     { return _Base_type::is_always_equal::value; }
 
+    __attribute__((__always_inline__))
     static constexpr bool _S_nothrow_move()
     { return _S_propagate_on_move_assign() || _S_always_equal(); }
 
@@ -125,30 +134,38 @@ template<typename _Alloc, typename = typename _Alloc::value_type>
     typedef typename _Alloc::size_type              size_type;
     typedef typename _Alloc::difference_type        difference_type;
 
-    _GLIBCXX_NODISCARD static pointer
+    __attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+    static pointer
     allocate(_Alloc& __a, size_type __n)
     { return __a.allocate(__n); }
 
     template<typename _Hint>
-      _GLIBCXX_NODISCARD static pointer
+      __attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+      static pointer
       allocate(_Alloc& __a, size_type __n, _Hint __hint)
       { return __a.allocate(__n, __hint); }
 
+    __attribute__((__always_inline__))
     static void deallocate(_Alloc& __a, pointer __p, size_type __n)
     { __a.deallocate(__p, __n); }
 
     template<typename _Tp>
+      __attribute__((__always_inline__))
       static void construct(_Alloc& __a, pointer __p, const _Tp& __arg)
       { __a.construct(__p, __arg); }
 
+    __attribute__((__always_inline__))
     static void destroy(_Alloc& __a, pointer __p)
     { __a.destroy(__p); }
 
+    __attribute__((__always_inline__))
     static size_type max_size(const _Alloc& __a)
     { return __a.max_size(); }
 
+    __attribute__((__always_inline__))
     static const _Alloc& _S_select_on_copy(const _Alloc& __a) { return __a; }
 
+    __attribute__((__always_inline__))
     static void _S_on_swap(_Alloc& __a, _Alloc& __b)
     {
       // _GLIBCXX_RESOLVE_LIB_DEFECTS