libstdc++: Inline std::exception_ptr members [PR 90295]

This inlines most members of std::exception_ptr so that all operations
on a null exception_ptr can be optimized away. This benefits code like
std::future and coroutines where an exception_ptr object is present to
cope with exceptional cases, but is usually not used and remains null.

Since those functions were previously non-inline we have to continue to
export them from the library, for objects that were compiled against the
old headers and expect to find definitions in the library.

In order to inline the copy constructor and destructor we need to export
the _M_addref() and _M_release() members that increment/decrement the
reference count when copying/destroying a non-null exception_ptr. The
copy ctor and dtor check for null and don't call _M_addref and
_M_release unless they need to. The checks for null pointers in
_M_addref and _M_release are still needed because old code might call
them without checking for null first. But we can use __builtin_expect to
predict that they are usually called for the non-null case.

libstdc++-v3/ChangeLog:

	PR libstdc++/90295
	* config/abi/pre/gnu.ver (CXXABI_1.3.13): New symbol version.
	(exception_ptr::_M_addref(), exception_ptr::_M_release()):
	Export symbols.
	* libsupc++/eh_ptr.cc (exception_ptr::exception_ptr()):
	Remove out-of-line definition.
	(exception_ptr::exception_ptr(const exception_ptr&)):
	Likewise.
	(exception_ptr::~exception_ptr()): Likewise.
	(exception_ptr::operator=(const exception_ptr&)):
	Likewise.
	(exception_ptr::swap(exception_ptr&)): Likewise.
	(exception_ptr::_M_addref()): Add branch prediction.
	* libsupc++/exception_ptr.h (exception_ptr::operator bool):
	Add noexcept.
	[!_GLIBCXX_EH_PTR_COMPAT] (operator==, operator!=): Define
	inline as hidden friends. Remove declarations at namespace
	scope.
	(exception_ptr::exception_ptr()): Define inline.
	(exception_ptr::exception_ptr(const exception_ptr&)):
	Likewise.
	(exception_ptr::~exception_ptr()): Likewise.
	(exception_ptr::operator=(const exception_ptr&)):
	Likewise.
	(exception_ptr::swap(exception_ptr&)): Likewise.
	* testsuite/util/testsuite_abi.cc: Add CXXABI_1.3.13.
	* testsuite/18_support/exception_ptr/90295.cc: New test.

5 files changed, 143 insertions, 53 deletions

diff --git a/libstdc++-v3/config/abi/pre/gnu.ver b/libstdc++-v3/config/abi/pre/gnu.ver
index 6a2b2da..a7f437a 100644
--- a/libstdc++-v3/config/abi/pre/gnu.ver
+++ b/libstdc++-v3/config/abi/pre/gnu.ver
@@ -2625,6 +2625,15 @@ CXXABI_1.3.12 {
 
 } CXXABI_1.3.11;
 
+CXXABI_1.3.13 {
+
+    # std::exception_ptr::_M_addref()
+    _ZNSt15__exception_ptr13exception_ptr9_M_addrefEv;
+    # std::exception_ptr::_M_release()
+    _ZNSt15__exception_ptr13exception_ptr10_M_releaseEv;
+
+} CXXABI_1.3.12;
+
 # Symbols in the support library (libsupc++) supporting transactional memory.
 CXXABI_TM_1 {
 
diff --git a/libstdc++-v3/libsupc++/eh_ptr.cc b/libstdc++-v3/libsupc++/eh_ptr.cc
index 0dc06f3..c41bdca 100644
--- a/libstdc++-v3/libsupc++/eh_ptr.cc
+++ b/libstdc++-v3/libsupc++/eh_ptr.cc
@@ -61,41 +61,18 @@ static_assert( adjptr<__cxa_exception>()
 #endif
 }
 
-std::__exception_ptr::exception_ptr::exception_ptr() noexcept
-: _M_exception_object(0) { }
-
-
 std::__exception_ptr::exception_ptr::exception_ptr(void* obj) noexcept
 : _M_exception_object(obj)  { _M_addref(); }
 
 
 std::__exception_ptr::exception_ptr::exception_ptr(__safe_bool) noexcept
-: _M_exception_object(0) { }
-
-
-std::__exception_ptr::
-exception_ptr::exception_ptr(const exception_ptr& other) noexcept
-: _M_exception_object(other._M_exception_object)
-{ _M_addref(); }
-
-
-std::__exception_ptr::exception_ptr::~exception_ptr() noexcept
-{ _M_release(); }
-
-
-std::__exception_ptr::exception_ptr&
-std::__exception_ptr::
-exception_ptr::operator=(const exception_ptr& other) noexcept
-{
-  exception_ptr(other).swap(*this);
-  return *this;
-}
+: _M_exception_object(nullptr) { }
 
 
 void
 std::__exception_ptr::exception_ptr::_M_addref() noexcept
 {
-  if (_M_exception_object)
+  if (__builtin_expect(_M_exception_object != nullptr, true))
     {
       __cxa_refcounted_exception *eh =
 	__get_refcounted_exception_header_from_obj (_M_exception_object);
@@ -107,7 +84,7 @@ std::__exception_ptr::exception_ptr::_M_addref() noexcept
 void
 std::__exception_ptr::exception_ptr::_M_release() noexcept
 {
-  if (_M_exception_object)
+  if (__builtin_expect(_M_exception_object != nullptr, true))
     {
       __cxa_refcounted_exception *eh =
 	__get_refcounted_exception_header_from_obj (_M_exception_object);
@@ -117,7 +94,7 @@ std::__exception_ptr::exception_ptr::_M_release() noexcept
 	    eh->exc.exceptionDestructor (_M_exception_object);
 
           __cxa_free_exception (_M_exception_object);
-          _M_exception_object = 0;
+          _M_exception_object = nullptr;
         }
     }
 }
@@ -128,14 +105,6 @@ std::__exception_ptr::exception_ptr::_M_get() const noexcept
 { return _M_exception_object; }
 
 
-void
-std::__exception_ptr::exception_ptr::swap(exception_ptr &other) noexcept
-{
-  void *tmp = _M_exception_object;
-  _M_exception_object = other._M_exception_object;
-  other._M_exception_object = tmp;
-}
-
 
 // Retained for compatibility with CXXABI_1.3.
 void
@@ -145,16 +114,15 @@ std::__exception_ptr::exception_ptr::_M_safe_bool_dummy() noexcept { }
 // Retained for compatibility with CXXABI_1.3.
 bool
 std::__exception_ptr::exception_ptr::operator!() const noexcept
-{ return _M_exception_object == 0; }
+{ return _M_exception_object == nullptr; }
 
 
 // Retained for compatibility with CXXABI_1.3.
 std::__exception_ptr::exception_ptr::operator __safe_bool() const noexcept
 {
-  return _M_exception_object ? &exception_ptr::_M_safe_bool_dummy : 0;
+  return _M_exception_object ? &exception_ptr::_M_safe_bool_dummy : nullptr;
 }
 
-
 const std::type_info*
 std::__exception_ptr::exception_ptr::__cxa_exception_type() const noexcept
 {
@@ -162,15 +130,17 @@ std::__exception_ptr::exception_ptr::__cxa_exception_type() const noexcept
   return eh->exceptionType;
 }
 
-
-bool std::__exception_ptr::operator==(const exception_ptr& lhs,
-				      const exception_ptr& rhs) noexcept
+// Retained for compatibility with CXXABI_1.3.12.
+bool
+std::__exception_ptr::operator==(const exception_ptr& lhs,
+				 const exception_ptr& rhs) noexcept
 { return lhs._M_exception_object == rhs._M_exception_object; }
 
-
-bool std::__exception_ptr::operator!=(const exception_ptr& lhs,
-				      const exception_ptr& rhs) noexcept
-{ return !(lhs == rhs);}
+// Retained for compatibility with CXXABI_1.3.12.
+bool
+std::__exception_ptr::operator!=(const exception_ptr& lhs,
+				 const exception_ptr& rhs) noexcept
+{ return !(lhs == rhs); }
 
 
 std::exception_ptr
diff --git a/libstdc++-v3/libsupc++/exception_ptr.h b/libstdc++-v3/libsupc++/exception_ptr.h
index fb38a3d..a053122 100644
--- a/libstdc++-v3/libsupc++/exception_ptr.h
+++ b/libstdc++-v3/libsupc++/exception_ptr.h
@@ -100,12 +100,12 @@ namespace std
 
 #if __cplusplus >= 201103L
       exception_ptr(nullptr_t) noexcept
-      : _M_exception_object(0)
+      : _M_exception_object(nullptr)
       { }
 
       exception_ptr(exception_ptr&& __o) noexcept
       : _M_exception_object(__o._M_exception_object)
-      { __o._M_exception_object = 0; }
+      { __o._M_exception_object = nullptr; }
 #endif
 
 #if (__cplusplus < 201103L) || defined (_GLIBCXX_EH_PTR_COMPAT)
@@ -142,21 +142,82 @@ namespace std
 #endif
 
 #if __cplusplus >= 201103L
-      explicit operator bool() const
+      explicit operator bool() const noexcept
       { return _M_exception_object; }
 #endif
 
-      friend bool 
+#ifdef _GLIBCXX_EH_PTR_COMPAT
+      friend bool
       operator==(const exception_ptr&, const exception_ptr&)
 	_GLIBCXX_USE_NOEXCEPT __attribute__ ((__pure__));
+#elif __cpp_impl_three_way_comparison >= 201907L
+      friend bool
+      operator==(const exception_ptr&, const exception_ptr&) noexcept = default;
+#else
+      friend bool
+      operator==(const exception_ptr& __x, const exception_ptr& __y)
+      _GLIBCXX_USE_NOEXCEPT
+      { return __x._M_exception_object == __y._M_exception_object; }
+
+      friend bool
+      operator!=(const exception_ptr& __x, const exception_ptr& __y)
+      _GLIBCXX_USE_NOEXCEPT
+      { return __x._M_exception_object != __y._M_exception_object; }
+#endif
 
       const class std::type_info*
       __cxa_exception_type() const _GLIBCXX_USE_NOEXCEPT
 	__attribute__ ((__pure__));
     };
 
-    /// @relates exception_ptr @{
+#ifndef _GLIBCXX_EH_PTR_COMPAT
+    inline
+#endif
+    exception_ptr::exception_ptr() _GLIBCXX_NOEXCEPT
+    : _M_exception_object(0)
+    { }
+
+#ifndef _GLIBCXX_EH_PTR_COMPAT
+    inline
+#endif
+    exception_ptr::exception_ptr(const exception_ptr& other) _GLIBCXX_NOEXCEPT
+    : _M_exception_object(other._M_exception_object)
+    {
+      if (_M_exception_object)
+	_M_addref();
+    }
+
+#ifndef _GLIBCXX_EH_PTR_COMPAT
+    inline
+#endif
+    exception_ptr::~exception_ptr() _GLIBCXX_USE_NOEXCEPT
+    {
+      if (_M_exception_object)
+	_M_release();
+    }
+
+#ifndef _GLIBCXX_EH_PTR_COMPAT
+    inline
+#endif
+    exception_ptr&
+    exception_ptr::operator=(const exception_ptr& other) _GLIBCXX_USE_NOEXCEPT
+    {
+      exception_ptr(other).swap(*this);
+      return *this;
+    }
 
+#ifndef _GLIBCXX_EH_PTR_COMPAT
+    inline
+#endif
+    void
+    exception_ptr::swap(exception_ptr &other) _GLIBCXX_USE_NOEXCEPT
+    {
+      void *tmp = _M_exception_object;
+      _M_exception_object = other._M_exception_object;
+      other._M_exception_object = tmp;
+    }
+
+#ifdef _GLIBCXX_EH_PTR_COMPAT
     bool 
     operator==(const exception_ptr&, const exception_ptr&)
       _GLIBCXX_USE_NOEXCEPT __attribute__ ((__pure__));
@@ -164,13 +225,13 @@ namespace std
     bool 
     operator!=(const exception_ptr&, const exception_ptr&)
       _GLIBCXX_USE_NOEXCEPT __attribute__ ((__pure__));
+#endif
 
+    /// @relates exception_ptr
     inline void
     swap(exception_ptr& __lhs, exception_ptr& __rhs)
     { __lhs.swap(__rhs); }
 
-    // @}
-
     /// @cond undocumented
     template<typename _Ex>
       inline void
diff --git a/libstdc++-v3/testsuite/18_support/exception_ptr/90295.cc b/libstdc++-v3/testsuite/18_support/exception_ptr/90295.cc
new file mode 100644
index 0000000..fa62ff1
--- /dev/null
+++ b/libstdc++-v3/testsuite/18_support/exception_ptr/90295.cc
@@ -0,0 +1,49 @@
+// Copyright (C) 2020 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-do compile { target c++11 } }
+// { dg-options "-O1 -g0" }
+// { dg-final { scan-assembler-not "St15__exception_ptr13exception_ptr" } }
+
+#include <exception>
+
+void
+test01()
+{
+  // PR libstdc++/90295
+  // Operations on null exception_ptr objects should be optimised away.
+
+  std::exception_ptr p1;
+  if (!(p1 == nullptr))
+    std::rethrow_exception(p1);
+
+  std::exception_ptr p2 = p1;
+  if (!(p2 == p1))
+    std::rethrow_exception(p2);
+
+  p1 = p2;
+  if (p1 != p2)
+    std::rethrow_exception(p1);
+
+  swap(p1, p2);
+  if (nullptr != p1)
+    std::rethrow_exception(p1);
+
+  p1 = std::exception_ptr(nullptr);
+  if (!(p1 == p2))
+    std::rethrow_exception(p1);
+}
diff --git a/libstdc++-v3/testsuite/util/testsuite_abi.cc b/libstdc++-v3/testsuite/util/testsuite_abi.cc
index f4bd319..33b9ec1 100644
--- a/libstdc++-v3/testsuite/util/testsuite_abi.cc
+++ b/libstdc++-v3/testsuite/util/testsuite_abi.cc
@@ -225,6 +225,7 @@ check_version(symbol& test, bool added)
       known_versions.push_back("CXXABI_1.3.10");
       known_versions.push_back("CXXABI_1.3.11");
       known_versions.push_back("CXXABI_1.3.12");
+      known_versions.push_back("CXXABI_1.3.13");
       known_versions.push_back("CXXABI_TM_1");
       known_versions.push_back("CXXABI_FLOAT128");
     }
@@ -246,7 +247,7 @@ check_version(symbol& test, bool added)
 	  // XXX remove next line when GLIBCXX_3.4.30 is added and baselines
 	  // have been regenerated to include GLIBCXX_LDBL_3.4.29 symbols:
 		     || test.version_name == "GLIBCXX_LDBL_3.4.29"
-		     || test.version_name == "CXXABI_1.3.12"
+		     || test.version_name == "CXXABI_1.3.13"
 		     || test.version_name == "CXXABI_FLOAT128"
 		     || test.version_name == "CXXABI_TM_1");
       if (added && !latestp)