libitm: Conversion to c++11 atomics.

        * local_atomic: New file.
        * libitm_i.h: Include it.
        (gtm_thread::shared_state): Use atomic template.
        * beginend.cc (GTM::gtm_clock): Use atomic template.
        (global_tid): Use atomic template if 64-bit atomics available.
        (gtm_thread::gtm_thread): Update shared_state access.
        (gtm_thread::trycommit): Likewise.
        (choose_code_path): Update global_tid access.
        * method-gl.cc (gl_mg::orec): Use atomic template.  Update all users.
        * stmlock.h (GTM::gtm_clock): Use atomic template.
        (gtm_get_clock, gtm_inc_clock): Update accesses.
        * config/linux/rwlock.cc (gtm_rwlock::read_lock): Remove
        redundant __sync_synchronize after atomic shared_state access.
        * config/posix/rwlock.cc (gtm_rwlock::read_lock): Likewise.
        (gtm_rwlock::write_lock_generic): Likewise.
        (gtm_rwlock::read_unlock): Likewise.
        * config/alpha/target.h (atomic_read_barrier): Remove.
        (atomic_write_barrier): Remove.
        * config/x86/target.h (atomic_read_barrier): Remove.
        (atomic_write_barrier): Remove.

From-SVN: r182294

10 files changed, 1996 insertions, 102 deletions

diff --git a/libitm/ChangeLog b/libitm/ChangeLog
index 3358464..cfb959d 100644
--- a/libitm/ChangeLog
+++ b/libitm/ChangeLog
@@ -1,3 +1,26 @@
+2011-12-13  Richard Henderson  <rth@redhat.com>
+
+	* local_atomic: New file.
+	* libitm_i.h: Include it.
+	(gtm_thread::shared_state): Use atomic template.
+	* beginend.cc (GTM::gtm_clock): Use atomic template.
+	(global_tid): Use atomic template if 64-bit atomics available.
+	(gtm_thread::gtm_thread): Update shared_state access.
+	(gtm_thread::trycommit): Likewise.
+	(choose_code_path): Update global_tid access.
+	* method-gl.cc (gl_mg::orec): Use atomic template.  Update all users.
+	* stmlock.h (GTM::gtm_clock): Use atomic template.
+	(gtm_get_clock, gtm_inc_clock): Update accesses.
+	* config/linux/rwlock.cc (gtm_rwlock::read_lock): Remove
+	redundant __sync_synchronize after atomic shared_state access.
+	* config/posix/rwlock.cc (gtm_rwlock::read_lock): Likewise.
+	(gtm_rwlock::write_lock_generic): Likewise.
+	(gtm_rwlock::read_unlock): Likewise.
+	* config/alpha/target.h (atomic_read_barrier): Remove.
+	(atomic_write_barrier): Remove.
+	* config/x86/target.h (atomic_read_barrier): Remove.
+	(atomic_write_barrier): Remove.
+
 2011-11-30  Richard Henderson  <rth@redhat.com>
 
 	* libitm_i.h (GTM_longjmp): Swap first and second arguments.
diff --git a/libitm/beginend.cc b/libitm/beginend.cc
index d8ef651..bcc8516 100644
--- a/libitm/beginend.cc
+++ b/libitm/beginend.cc
@@ -37,12 +37,18 @@ gtm_thread *GTM::gtm_thread::list_of_threads = 0;
 unsigned GTM::gtm_thread::number_of_threads = 0;
 
 gtm_stmlock GTM::gtm_stmlock_array[LOCK_ARRAY_SIZE];
-gtm_version GTM::gtm_clock;
+atomic<gtm_version> GTM::gtm_clock;
 
 /* ??? Move elsewhere when we figure out library initialization.  */
 uint64_t GTM::gtm_spin_count_var = 1000;
 
+#ifdef HAVE_64BIT_SYNC_BUILTINS
+static atomic<_ITM_transactionId_t> global_tid;
+#else
 static _ITM_transactionId_t global_tid;
+static pthread_mutex_t global_tid_lock = PTHREAD_MUTEX_INITIALIZER;
+#endif
+
 
 // Provides a on-thread-exit callback used to release per-thread data.
 static pthread_key_t thr_release_key;
@@ -114,7 +120,7 @@ GTM::gtm_thread::gtm_thread ()
   // This object's memory has been set to zero by operator new, so no need
   // to initialize any of the other primitive-type members that do not have
   // constructors.
-  shared_state = ~(typeof shared_state)0;
+  shared_state.store(-1, memory_order_relaxed);
 
   // Register this transaction with the list of all threads' transactions.
   serial_lock.write_lock ();
@@ -132,13 +138,8 @@ GTM::gtm_thread::gtm_thread ()
     GTM_fatal("Setting thread release TLS key failed.");
 }
 
-
-
-#ifndef HAVE_64BIT_SYNC_BUILTINS
-static pthread_mutex_t global_tid_lock = PTHREAD_MUTEX_INITIALIZER;
-#endif
-
-static inline uint32_t choose_code_path(uint32_t prop, abi_dispatch *disp)
+static inline uint32_t
+choose_code_path(uint32_t prop, abi_dispatch *disp)
 {
   if ((prop & pr_uninstrumentedCode) && disp->can_run_uninstrumented_code())
     return a_runUninstrumentedCode;
@@ -258,7 +259,7 @@ GTM::gtm_thread::begin_transaction (uint32_t prop, const gtm_jmpbuf *jb)
   else
     {
 #ifdef HAVE_64BIT_SYNC_BUILTINS
-      tx->id = __sync_add_and_fetch (&global_tid, tid_block_size);
+      tx->id = global_tid.fetch_add(tid_block_size, memory_order_relaxed);
       tx->local_tid = tx->id + 1;
 #else
       pthread_mutex_lock (&global_tid_lock);
@@ -480,7 +481,7 @@ GTM::gtm_thread::trycommit ()
 	      it = it->next_thread)
 	    {
 	      if (it == this) continue;
-	      while (it->shared_state < priv_time)
+	      while (it->shared_state.load(memory_order_relaxed) < priv_time)
 		cpu_relax();
 	    }
 	}
diff --git a/libitm/config/alpha/target.h b/libitm/config/alpha/target.h
index 121546f..12b1d89 100644
--- a/libitm/config/alpha/target.h
+++ b/libitm/config/alpha/target.h
@@ -45,16 +45,4 @@ cpu_relax (void)
   __asm volatile ("" : : : "memory");
 }
 
-static inline void
-atomic_read_barrier (void)
-{
-  __sync_synchronize ();
-}
-
-static inline void
-atomic_write_barrier (void)
-{
-  __asm volatile ("wmb" : : : "memory");
-}
-
 } // namespace GTM
diff --git a/libitm/config/linux/rwlock.cc b/libitm/config/linux/rwlock.cc
index c1e935e..3471049 100644
--- a/libitm/config/linux/rwlock.cc
+++ b/libitm/config/linux/rwlock.cc
@@ -36,10 +36,9 @@ gtm_rwlock::read_lock (gtm_thread *tx)
   for (;;)
     {
       // Fast path: first announce our intent to read, then check for
-      // conflicting intents to write. The barrier makes sure that this
-      // happens in exactly this order.
+      // conflicting intents to write.  Note that direct assignment to
+      // an atomic object is memory_order_seq_cst.
       tx->shared_state = 0;
-      __sync_synchronize();
       if (likely(writers == 0))
 	return;
 
@@ -51,8 +50,7 @@ gtm_rwlock::read_lock (gtm_thread *tx)
       // We need the barrier here for the same reason that we need it in
       // read_unlock().
       // TODO Potentially too many wake-ups. See comments in read_unlock().
-      tx->shared_state = ~(typeof tx->shared_state)0;
-      __sync_synchronize();
+      tx->shared_state = -1;
       if (writer_readers > 0)
 	{
 	  writer_readers = 0;
@@ -71,7 +69,7 @@ gtm_rwlock::read_lock (gtm_thread *tx)
 	  // are no writers anymore after the barrier because this pending
 	  // store could then lead to lost wake-ups at other readers.
 	  readers = 1;
-	  __sync_synchronize();
+	  atomic_thread_fence(memory_order_acq_rel);
 	  if (writers)
 	    futex_wait(&readers, 1);
 	}
diff --git a/libitm/config/posix/rwlock.cc b/libitm/config/posix/rwlock.cc
index f379383..e1e3dcf 100644
--- a/libitm/config/posix/rwlock.cc
+++ b/libitm/config/posix/rwlock.cc
@@ -53,10 +53,9 @@ void
 gtm_rwlock::read_lock (gtm_thread *tx)
 {
   // Fast path: first announce our intent to read, then check for conflicting
-  // intents to write. The barrier makes sure that this happens in exactly
-  // this order.
+  // intents to write.  Note that direct assignment to an atomic object
+  // is memory_order_seq_cst.
   tx->shared_state = 0;
-  __sync_synchronize();
   unsigned int sum = this->summary;
   if (likely(!(sum & (a_writer | w_writer))))
     return;
@@ -69,7 +68,7 @@ gtm_rwlock::read_lock (gtm_thread *tx)
   // to happen before we leave the slow path and before we wait for any
   // writer).
   // ??? Add a barrier to enforce early visibility of this?
-  tx->shared_state = ~(typeof tx->shared_state)0;
+  tx->shared_state.store(-1, memory_order_relaxed);
 
   pthread_mutex_lock (&this->mutex);
 
@@ -101,7 +100,7 @@ gtm_rwlock::read_lock (gtm_thread *tx)
     }
 
   // Otherwise we can acquire the lock for read.
-  tx->shared_state = 0;
+  tx->shared_state.store(0, memory_order_relaxed);
 
   pthread_mutex_unlock(&this->mutex);
 }
@@ -153,11 +152,11 @@ gtm_rwlock::write_lock_generic (gtm_thread *tx)
   // sure that we first set our write intent and check for active readers
   // after that, in strictly this order (similar to the barrier in the fast
   // path of read_lock()).
-  __sync_synchronize();
+  atomic_thread_fence(memory_order_acq_rel);
 
   // If this is an upgrade, we are not a reader anymore.
   if (tx != 0)
-    tx->shared_state = ~(typeof tx->shared_state)0;
+    tx->shared_state.store(-1, memory_order_relaxed);
 
   // Count the number of active readers to be able to decrease the number of
   // wake-ups and wait calls that are necessary.
@@ -194,7 +193,7 @@ gtm_rwlock::write_lock_generic (gtm_thread *tx)
 	  it = it->next_thread)
 	{
 	  // Don't count ourself if this is an upgrade.
-	  if (it->shared_state != ~(typeof it->shared_state)0)
+	  if (it->shared_state.load(memory_order_relaxed) != -1)
 	    readers++;
 	}
 
@@ -236,8 +235,7 @@ gtm_rwlock::write_upgrade (gtm_thread *tx)
 void
 gtm_rwlock::read_unlock (gtm_thread *tx)
 {
-  tx->shared_state = ~(typeof tx->shared_state)0;
-  __sync_synchronize();
+  tx->shared_state = -1;
   unsigned int sum = this->summary;
   if (likely(!(sum & (a_writer | w_writer))))
     return;
diff --git a/libitm/config/x86/target.h b/libitm/config/x86/target.h
index 197faeb..a59608f 100644
--- a/libitm/config/x86/target.h
+++ b/libitm/config/x86/target.h
@@ -66,20 +66,6 @@ cpu_relax (void)
   __asm volatile ("rep; nop" : : : "memory");
 }
 
-static inline void
-atomic_read_barrier (void)
-{
-  /* x86 is a strong memory ordering machine.  */
-  __asm volatile ("" : : : "memory");
-}
-
-static inline void
-atomic_write_barrier (void)
-{
-  /* x86 is a strong memory ordering machine.  */
-  __asm volatile ("" : : : "memory");
-}
-
 } // namespace GTM
 
 // We'll be using some of the cpu builtins, and their associated types.
diff --git a/libitm/libitm_i.h b/libitm/libitm_i.h
index ff6884e..d57872e 100644
--- a/libitm/libitm_i.h
+++ b/libitm/libitm_i.h
@@ -37,6 +37,7 @@
 #include <string.h>
 #include <unwind.h>
 #include "local_type_traits"
+#include "local_atomic"
 
 #include "common.h"
 
@@ -206,7 +207,7 @@ struct gtm_thread
 
   // If this transaction is inactive, shared_state is ~0. Otherwise, this is
   // an active or serial transaction.
-  gtm_word shared_state;
+  atomic<gtm_word> shared_state;
 
   // The lock that provides access to serial mode.  Non-serialized
   // transactions acquire read locks; a serialized transaction aquires
diff --git a/libitm/local_atomic b/libitm/local_atomic
new file mode 100644
index 0000000..e664463
--- /dev/null
+++ b/libitm/local_atomic
@@ -0,0 +1,1903 @@
+// -*- C++ -*- header.
+
+// Copyright (C) 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+// ????????????????????????????????????????????????????????????????????
+//
+// This is a copy of the libstdc++ header, with the trivial modification
+// of ignoring the c++config.h include.  If and when the top-level build is
+// fixed so that target libraries can be built using the newly built, we can
+// delete this file.
+//
+// ????????????????????????????????????????????????????????????????????
+
+/** @file include/atomic
+ *  This is a Standard C++ Library header.
+ */
+
+// Based on "C++ Atomic Types and Operations" by Hans Boehm and Lawrence Crowl.
+// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2007/n2427.html
+
+#ifndef _GLIBCXX_ATOMIC
+#define _GLIBCXX_ATOMIC 1
+
+// #pragma GCC system_header
+
+// #ifndef __GXX_EXPERIMENTAL_CXX0X__
+// # include <bits/c++0x_warning.h>
+// #endif
+
+// #include <bits/atomic_base.h>
+
+namespace std // _GLIBCXX_VISIBILITY(default)
+{
+// _GLIBCXX_BEGIN_NAMESPACE_VERSION
+
+  /**
+   * @defgroup atomics Atomics
+   *
+   * Components for performing atomic operations.
+   * @{
+   */
+
+  /// Enumeration for memory_order
+  typedef enum memory_order
+    {
+      memory_order_relaxed,
+      memory_order_consume,
+      memory_order_acquire,
+      memory_order_release,
+      memory_order_acq_rel,
+      memory_order_seq_cst
+    } memory_order;
+
+  inline memory_order
+  __calculate_memory_order(memory_order __m) noexcept
+  {
+    const bool __cond1 = __m == memory_order_release;
+    const bool __cond2 = __m == memory_order_acq_rel;
+    memory_order __mo1(__cond1 ? memory_order_relaxed : __m);
+    memory_order __mo2(__cond2 ? memory_order_acquire : __mo1);
+    return __mo2;
+  }
+
+  inline void
+  atomic_thread_fence(memory_order __m) noexcept
+  {
+    __atomic_thread_fence (__m);
+  }
+
+  inline void
+  atomic_signal_fence(memory_order __m) noexcept
+  {
+    __atomic_thread_fence (__m);
+  }
+
+  /// kill_dependency
+  template<typename _Tp>
+    inline _Tp
+    kill_dependency(_Tp __y) noexcept
+    {
+      _Tp __ret(__y);
+      return __ret;
+    }
+
+  /// Lock-free Property
+
+
+#define ATOMIC_BOOL_LOCK_FREE		__GCC_ATOMIC_BOOL_LOCK_FREE
+#define ATOMIC_CHAR_LOCK_FREE		__GCC_ATOMIC_CHAR_LOCK_FREE
+#define ATOMIC_WCHAR_T_LOCK_FREE	__GCC_ATOMIC_WCHAR_T_LOCK_FREE
+#define ATOMIC_CHAR16_T_LOCK_FREE	__GCC_ATOMIC_CHAR16_T_LOCK_FREE
+#define ATOMIC_CHAR32_T_LOCK_FREE	__GCC_ATOMIC_CHAR32_T_LOCK_FREE
+#define ATOMIC_SHORT_LOCK_FREE		__GCC_ATOMIC_SHORT_LOCK_FREE
+#define ATOMIC_INT_LOCK_FREE		__GCC_ATOMIC_INT_LOCK_FREE
+#define ATOMIC_LONG_LOCK_FREE		__GCC_ATOMIC_LONG_LOCK_FREE
+#define ATOMIC_LLONG_LOCK_FREE		__GCC_ATOMIC_LLONG_LOCK_FREE
+#define ATOMIC_POINTER_LOCK_FREE	__GCC_ATOMIC_POINTER_LOCK_FREE
+
+  // Base types for atomics.
+  template<typename _IntTp>
+    struct __atomic_base;
+
+  /// atomic_char
+  typedef __atomic_base<char>  	       		atomic_char;
+
+  /// atomic_schar
+  typedef __atomic_base<signed char>	     	atomic_schar;
+
+  /// atomic_uchar
+  typedef __atomic_base<unsigned char>		atomic_uchar;
+
+  /// atomic_short
+  typedef __atomic_base<short>			atomic_short;
+
+  /// atomic_ushort
+  typedef __atomic_base<unsigned short>	 	atomic_ushort;
+
+  /// atomic_int
+  typedef __atomic_base<int>  	       		atomic_int;
+
+  /// atomic_uint
+  typedef __atomic_base<unsigned int>	     	atomic_uint;
+
+  /// atomic_long
+  typedef __atomic_base<long>  	       		atomic_long;
+
+  /// atomic_ulong
+  typedef __atomic_base<unsigned long>		atomic_ulong;
+
+  /// atomic_llong
+  typedef __atomic_base<long long>  		atomic_llong;
+
+  /// atomic_ullong
+  typedef __atomic_base<unsigned long long> 	atomic_ullong;
+
+  /// atomic_wchar_t
+  typedef __atomic_base<wchar_t>  		atomic_wchar_t;
+
+  /// atomic_char16_t
+  typedef __atomic_base<char16_t>  		atomic_char16_t;
+
+  /// atomic_char32_t
+  typedef __atomic_base<char32_t>  		atomic_char32_t;
+
+  /// atomic_char32_t
+  typedef __atomic_base<char32_t>  		atomic_char32_t;
+
+
+  /// atomic_int_least8_t
+  typedef __atomic_base<int_least8_t>  		atomic_int_least8_t;
+
+  /// atomic_uint_least8_t
+  typedef __atomic_base<uint_least8_t>	       	atomic_uint_least8_t;
+
+  /// atomic_int_least16_t
+  typedef __atomic_base<int_least16_t>	       	atomic_int_least16_t;
+
+  /// atomic_uint_least16_t
+  typedef __atomic_base<uint_least16_t>	       	atomic_uint_least16_t;
+
+  /// atomic_int_least32_t
+  typedef __atomic_base<int_least32_t>	       	atomic_int_least32_t;
+
+  /// atomic_uint_least32_t
+  typedef __atomic_base<uint_least32_t>	       	atomic_uint_least32_t;
+
+  /// atomic_int_least64_t
+  typedef __atomic_base<int_least64_t>	       	atomic_int_least64_t;
+
+  /// atomic_uint_least64_t
+  typedef __atomic_base<uint_least64_t>	       	atomic_uint_least64_t;
+
+
+  /// atomic_int_fast8_t
+  typedef __atomic_base<int_fast8_t>  		atomic_int_fast8_t;
+
+  /// atomic_uint_fast8_t
+  typedef __atomic_base<uint_fast8_t>	      	atomic_uint_fast8_t;
+
+  /// atomic_int_fast16_t
+  typedef __atomic_base<int_fast16_t>	      	atomic_int_fast16_t;
+
+  /// atomic_uint_fast16_t
+  typedef __atomic_base<uint_fast16_t>	      	atomic_uint_fast16_t;
+
+  /// atomic_int_fast32_t
+  typedef __atomic_base<int_fast32_t>	      	atomic_int_fast32_t;
+
+  /// atomic_uint_fast32_t
+  typedef __atomic_base<uint_fast32_t>	      	atomic_uint_fast32_t;
+
+  /// atomic_int_fast64_t
+  typedef __atomic_base<int_fast64_t>	      	atomic_int_fast64_t;
+
+  /// atomic_uint_fast64_t
+  typedef __atomic_base<uint_fast64_t>	      	atomic_uint_fast64_t;
+
+
+  /// atomic_intptr_t
+  typedef __atomic_base<intptr_t>  	       	atomic_intptr_t;
+
+  /// atomic_uintptr_t
+  typedef __atomic_base<uintptr_t>  	       	atomic_uintptr_t;
+
+  /// atomic_size_t
+  typedef __atomic_base<size_t>	 	       	atomic_size_t;
+
+  /// atomic_intmax_t
+  typedef __atomic_base<intmax_t>  	       	atomic_intmax_t;
+
+  /// atomic_uintmax_t
+  typedef __atomic_base<uintmax_t>  	       	atomic_uintmax_t;
+
+  /// atomic_ptrdiff_t
+  typedef __atomic_base<ptrdiff_t>  	       	atomic_ptrdiff_t;
+
+
+#define ATOMIC_VAR_INIT(_VI) { _VI }
+
+  template<typename _Tp>
+    struct atomic;
+
+  template<typename _Tp>
+    struct atomic<_Tp*>;
+
+
+  /**
+   *  @brief Base type for atomic_flag.
+   *
+   *  Base type is POD with data, allowing atomic_flag to derive from
+   *  it and meet the standard layout type requirement. In addition to
+   *  compatibilty with a C interface, this allows different
+   *  implementations of atomic_flag to use the same atomic operation
+   *  functions, via a standard conversion to the __atomic_flag_base
+   *  argument.
+  */
+  // _GLIBCXX_BEGIN_EXTERN_C
+
+  struct __atomic_flag_base
+  {
+    bool _M_i;
+  };
+
+  // _GLIBCXX_END_EXTERN_C
+
+#define ATOMIC_FLAG_INIT { false }
+
+  /// atomic_flag
+  struct atomic_flag : public __atomic_flag_base
+  {
+    atomic_flag() noexcept = default;
+    ~atomic_flag() noexcept = default;
+    atomic_flag(const atomic_flag&) = delete;
+    atomic_flag& operator=(const atomic_flag&) = delete;
+    atomic_flag& operator=(const atomic_flag&) volatile = delete;
+
+    // Conversion to ATOMIC_FLAG_INIT.
+    atomic_flag(bool __i) noexcept : __atomic_flag_base({ __i }) { }
+
+    bool
+    test_and_set(memory_order __m = memory_order_seq_cst) noexcept
+    {
+      return __atomic_test_and_set (&_M_i, __m);
+    }
+
+    bool
+    test_and_set(memory_order __m = memory_order_seq_cst) volatile noexcept
+    {
+      return __atomic_test_and_set (&_M_i, __m);
+    }
+
+    void
+    clear(memory_order __m = memory_order_seq_cst) noexcept
+    {
+      // __glibcxx_assert(__m != memory_order_consume);
+      // __glibcxx_assert(__m != memory_order_acquire);
+      // __glibcxx_assert(__m != memory_order_acq_rel);
+
+      __atomic_clear (&_M_i, __m);
+    }
+
+    void
+    clear(memory_order __m = memory_order_seq_cst) volatile noexcept
+    {
+      // __glibcxx_assert(__m != memory_order_consume);
+      // __glibcxx_assert(__m != memory_order_acquire);
+      // __glibcxx_assert(__m != memory_order_acq_rel);
+
+      __atomic_clear (&_M_i, __m);
+    }
+  };
+
+
+  /// Base class for atomic integrals.
+  //
+  // For each of the integral types, define atomic_[integral type] struct
+  //
+  // atomic_bool     bool
+  // atomic_char     char
+  // atomic_schar    signed char
+  // atomic_uchar    unsigned char
+  // atomic_short    short
+  // atomic_ushort   unsigned short
+  // atomic_int      int
+  // atomic_uint     unsigned int
+  // atomic_long     long
+  // atomic_ulong    unsigned long
+  // atomic_llong    long long
+  // atomic_ullong   unsigned long long
+  // atomic_char16_t char16_t
+  // atomic_char32_t char32_t
+  // atomic_wchar_t  wchar_t
+  //
+  // NB: Assuming _ITp is an integral scalar type that is 1, 2, 4, or
+  // 8 bytes, since that is what GCC built-in functions for atomic
+  // memory access expect.
+  template<typename _ITp>
+    struct __atomic_base
+    {
+    private:
+      typedef _ITp 	__int_type;
+
+      __int_type 	_M_i;
+
+    public:
+      __atomic_base() noexcept = default;
+      ~__atomic_base() noexcept = default;
+      __atomic_base(const __atomic_base&) = delete;
+      __atomic_base& operator=(const __atomic_base&) = delete;
+      __atomic_base& operator=(const __atomic_base&) volatile = delete;
+
+      // Requires __int_type convertible to _M_i.
+      constexpr __atomic_base(__int_type __i) noexcept : _M_i (__i) { }
+
+      operator __int_type() const noexcept
+      { return load(); }
+
+      operator __int_type() const volatile noexcept
+      { return load(); }
+
+      __int_type
+      operator=(__int_type __i) noexcept
+      {
+	store(__i);
+	return __i;
+      }
+
+      __int_type
+      operator=(__int_type __i) volatile noexcept
+      {
+	store(__i);
+	return __i;
+      }
+
+      __int_type
+      operator++(int) noexcept
+      { return fetch_add(1); }
+
+      __int_type
+      operator++(int) volatile noexcept
+      { return fetch_add(1); }
+
+      __int_type
+      operator--(int) noexcept
+      { return fetch_sub(1); }
+
+      __int_type
+      operator--(int) volatile noexcept
+      { return fetch_sub(1); }
+
+      __int_type
+      operator++() noexcept
+      { return __atomic_add_fetch(&_M_i, 1, memory_order_seq_cst); }
+
+      __int_type
+      operator++() volatile noexcept
+      { return __atomic_add_fetch(&_M_i, 1, memory_order_seq_cst); }
+
+      __int_type
+      operator--() noexcept
+      { return __atomic_sub_fetch(&_M_i, 1, memory_order_seq_cst); }
+
+      __int_type
+      operator--() volatile noexcept
+      { return __atomic_sub_fetch(&_M_i, 1, memory_order_seq_cst); }
+
+      __int_type
+      operator+=(__int_type __i) noexcept
+      { return __atomic_add_fetch(&_M_i, __i, memory_order_seq_cst); }
+
+      __int_type
+      operator+=(__int_type __i) volatile noexcept
+      { return __atomic_add_fetch(&_M_i, __i, memory_order_seq_cst); }
+
+      __int_type
+      operator-=(__int_type __i) noexcept
+      { return __atomic_sub_fetch(&_M_i, __i, memory_order_seq_cst); }
+
+      __int_type
+      operator-=(__int_type __i) volatile noexcept
+      { return __atomic_sub_fetch(&_M_i, __i, memory_order_seq_cst); }
+
+      __int_type
+      operator&=(__int_type __i) noexcept
+      { return __atomic_and_fetch(&_M_i, __i, memory_order_seq_cst); }
+
+      __int_type
+      operator&=(__int_type __i) volatile noexcept
+      { return __atomic_and_fetch(&_M_i, __i, memory_order_seq_cst); }
+
+      __int_type
+      operator|=(__int_type __i) noexcept
+      { return __atomic_or_fetch(&_M_i, __i, memory_order_seq_cst); }
+
+      __int_type
+      operator|=(__int_type __i) volatile noexcept
+      { return __atomic_or_fetch(&_M_i, __i, memory_order_seq_cst); }
+
+      __int_type
+      operator^=(__int_type __i) noexcept
+      { return __atomic_xor_fetch(&_M_i, __i, memory_order_seq_cst); }
+
+      __int_type
+      operator^=(__int_type __i) volatile noexcept
+      { return __atomic_xor_fetch(&_M_i, __i, memory_order_seq_cst); }
+
+      bool
+      is_lock_free() const noexcept
+      { return __atomic_is_lock_free (sizeof (_M_i), &_M_i); }
+
+      bool
+      is_lock_free() const volatile noexcept
+      { return __atomic_is_lock_free (sizeof (_M_i), &_M_i); }
+
+      void
+      store(__int_type __i, memory_order __m = memory_order_seq_cst) noexcept
+      {
+	// __glibcxx_assert(__m != memory_order_acquire);
+	// __glibcxx_assert(__m != memory_order_acq_rel);
+	// __glibcxx_assert(__m != memory_order_consume);
+
+	__atomic_store_n(&_M_i, __i, __m);
+      }
+
+      void
+      store(__int_type __i,
+	    memory_order __m = memory_order_seq_cst) volatile noexcept
+      {
+	// __glibcxx_assert(__m != memory_order_acquire);
+	// __glibcxx_assert(__m != memory_order_acq_rel);
+	// __glibcxx_assert(__m != memory_order_consume);
+
+	__atomic_store_n(&_M_i, __i, __m);
+      }
+
+      __int_type
+      load(memory_order __m = memory_order_seq_cst) const noexcept
+      {
+	// __glibcxx_assert(__m != memory_order_release);
+	// __glibcxx_assert(__m != memory_order_acq_rel);
+
+	return __atomic_load_n(&_M_i, __m);
+      }
+
+      __int_type
+      load(memory_order __m = memory_order_seq_cst) const volatile noexcept
+      {
+	// __glibcxx_assert(__m != memory_order_release);
+	// __glibcxx_assert(__m != memory_order_acq_rel);
+
+	return __atomic_load_n(&_M_i, __m);
+      }
+
+      __int_type
+      exchange(__int_type __i,
+	       memory_order __m = memory_order_seq_cst) noexcept
+      {
+	return __atomic_exchange_n(&_M_i, __i, __m);
+      }
+
+
+      __int_type
+      exchange(__int_type __i,
+	       memory_order __m = memory_order_seq_cst) volatile noexcept
+      {
+	return __atomic_exchange_n(&_M_i, __i, __m);
+      }
+
+      bool
+      compare_exchange_weak(__int_type& __i1, __int_type __i2,
+			    memory_order __m1, memory_order __m2) noexcept
+      {
+	// __glibcxx_assert(__m2 != memory_order_release);
+	// __glibcxx_assert(__m2 != memory_order_acq_rel);
+	// __glibcxx_assert(__m2 <= __m1);
+
+	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1, __m1, __m2);
+      }
+
+      bool
+      compare_exchange_weak(__int_type& __i1, __int_type __i2,
+			    memory_order __m1,
+			    memory_order __m2) volatile noexcept
+      {
+	// __glibcxx_assert(__m2 != memory_order_release);
+	// __glibcxx_assert(__m2 != memory_order_acq_rel);
+	// __glibcxx_assert(__m2 <= __m1);
+
+	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1, __m1, __m2);
+      }
+
+      bool
+      compare_exchange_weak(__int_type& __i1, __int_type __i2,
+			    memory_order __m = memory_order_seq_cst) noexcept
+      {
+	return compare_exchange_weak(__i1, __i2, __m,
+				     __calculate_memory_order(__m));
+      }
+
+      bool
+      compare_exchange_weak(__int_type& __i1, __int_type __i2,
+		   memory_order __m = memory_order_seq_cst) volatile noexcept
+      {
+	return compare_exchange_weak(__i1, __i2, __m,
+				     __calculate_memory_order(__m));
+      }
+
+      bool
+      compare_exchange_strong(__int_type& __i1, __int_type __i2,
+			      memory_order __m1, memory_order __m2) noexcept
+      {
+	// __glibcxx_assert(__m2 != memory_order_release);
+	// __glibcxx_assert(__m2 != memory_order_acq_rel);
+	// __glibcxx_assert(__m2 <= __m1);
+
+	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0, __m1, __m2);
+      }
+
+      bool
+      compare_exchange_strong(__int_type& __i1, __int_type __i2,
+			      memory_order __m1,
+			      memory_order __m2) volatile noexcept
+      {
+	// __glibcxx_assert(__m2 != memory_order_release);
+	// __glibcxx_assert(__m2 != memory_order_acq_rel);
+	// __glibcxx_assert(__m2 <= __m1);
+
+	return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0, __m1, __m2);
+      }
+
+      bool
+      compare_exchange_strong(__int_type& __i1, __int_type __i2,
+			      memory_order __m = memory_order_seq_cst) noexcept
+      {
+	return compare_exchange_strong(__i1, __i2, __m,
+				       __calculate_memory_order(__m));
+      }
+
+      bool
+      compare_exchange_strong(__int_type& __i1, __int_type __i2,
+		 memory_order __m = memory_order_seq_cst) volatile noexcept
+      {
+	return compare_exchange_strong(__i1, __i2, __m,
+				       __calculate_memory_order(__m));
+      }
+
+      __int_type
+      fetch_add(__int_type __i,
+		memory_order __m = memory_order_seq_cst) noexcept
+      { return __atomic_fetch_add(&_M_i, __i, __m); }
+
+      __int_type
+      fetch_add(__int_type __i,
+		memory_order __m = memory_order_seq_cst) volatile noexcept
+      { return __atomic_fetch_add(&_M_i, __i, __m); }
+
+      __int_type
+      fetch_sub(__int_type __i,
+		memory_order __m = memory_order_seq_cst) noexcept
+      { return __atomic_fetch_sub(&_M_i, __i, __m); }
+
+      __int_type
+      fetch_sub(__int_type __i,
+		memory_order __m = memory_order_seq_cst) volatile noexcept
+      { return __atomic_fetch_sub(&_M_i, __i, __m); }
+
+      __int_type
+      fetch_and(__int_type __i,
+		memory_order __m = memory_order_seq_cst) noexcept
+      { return __atomic_fetch_and(&_M_i, __i, __m); }
+
+      __int_type
+      fetch_and(__int_type __i,
+		memory_order __m = memory_order_seq_cst) volatile noexcept
+      { return __atomic_fetch_and(&_M_i, __i, __m); }
+
+      __int_type
+      fetch_or(__int_type __i,
+	       memory_order __m = memory_order_seq_cst) noexcept
+      { return __atomic_fetch_or(&_M_i, __i, __m); }
+
+      __int_type
+      fetch_or(__int_type __i,
+	       memory_order __m = memory_order_seq_cst) volatile noexcept
+      { return __atomic_fetch_or(&_M_i, __i, __m); }
+
+      __int_type
+      fetch_xor(__int_type __i,
+		memory_order __m = memory_order_seq_cst) noexcept
+      { return __atomic_fetch_xor(&_M_i, __i, __m); }
+
+      __int_type
+      fetch_xor(__int_type __i,
+		memory_order __m = memory_order_seq_cst) volatile noexcept
+      { return __atomic_fetch_xor(&_M_i, __i, __m); }
+    };
+
+
+  /// Partial specialization for pointer types.
+  template<typename _PTp>
+    struct __atomic_base<_PTp*>
+    {
+    private:
+      typedef _PTp* 	__pointer_type;
+
+      __pointer_type 	_M_p;
+
+    public:
+      __atomic_base() noexcept = default;
+      ~__atomic_base() noexcept = default;
+      __atomic_base(const __atomic_base&) = delete;
+      __atomic_base& operator=(const __atomic_base&) = delete;
+      __atomic_base& operator=(const __atomic_base&) volatile = delete;
+
+      // Requires __pointer_type convertible to _M_p.
+      constexpr __atomic_base(__pointer_type __p) noexcept : _M_p (__p) { }
+
+      operator __pointer_type() const noexcept
+      { return load(); }
+
+      operator __pointer_type() const volatile noexcept
+      { return load(); }
+
+      __pointer_type
+      operator=(__pointer_type __p) noexcept
+      {
+	store(__p);
+	return __p;
+      }
+
+      __pointer_type
+      operator=(__pointer_type __p) volatile noexcept
+      {
+	store(__p);
+	return __p;
+      }
+
+      __pointer_type
+      operator++(int) noexcept
+      { return fetch_add(1); }
+
+      __pointer_type
+      operator++(int) volatile noexcept
+      { return fetch_add(1); }
+
+      __pointer_type
+      operator--(int) noexcept
+      { return fetch_sub(1); }
+
+      __pointer_type
+      operator--(int) volatile noexcept
+      { return fetch_sub(1); }
+
+      __pointer_type
+      operator++() noexcept
+      { return __atomic_add_fetch(&_M_p, 1, memory_order_seq_cst); }
+
+      __pointer_type
+      operator++() volatile noexcept
+      { return __atomic_add_fetch(&_M_p, 1, memory_order_seq_cst); }
+
+      __pointer_type
+      operator--() noexcept
+      { return __atomic_sub_fetch(&_M_p, 1, memory_order_seq_cst); }
+
+      __pointer_type
+      operator--() volatile noexcept
+      { return __atomic_sub_fetch(&_M_p, 1, memory_order_seq_cst); }
+
+      __pointer_type
+      operator+=(ptrdiff_t __d) noexcept
+      { return __atomic_add_fetch(&_M_p, __d, memory_order_seq_cst); }
+
+      __pointer_type
+      operator+=(ptrdiff_t __d) volatile noexcept
+      { return __atomic_add_fetch(&_M_p, __d, memory_order_seq_cst); }
+
+      __pointer_type
+      operator-=(ptrdiff_t __d) noexcept
+      { return __atomic_sub_fetch(&_M_p, __d, memory_order_seq_cst); }
+
+      __pointer_type
+      operator-=(ptrdiff_t __d) volatile noexcept
+      { return __atomic_sub_fetch(&_M_p, __d, memory_order_seq_cst); }
+
+      bool
+      is_lock_free() const noexcept
+      { return __atomic_is_lock_free (sizeof (_M_p), &_M_p); }
+
+      bool
+      is_lock_free() const volatile noexcept
+      { return __atomic_is_lock_free (sizeof (_M_p), &_M_p); }
+
+      void
+      store(__pointer_type __p,
+	    memory_order __m = memory_order_seq_cst) noexcept
+      {
+	// __glibcxx_assert(__m != memory_order_acquire);
+	// __glibcxx_assert(__m != memory_order_acq_rel);
+	// __glibcxx_assert(__m != memory_order_consume);
+
+	__atomic_store_n(&_M_p, __p, __m);
+      }
+
+      void
+      store(__pointer_type __p,
+	    memory_order __m = memory_order_seq_cst) volatile noexcept
+      {
+	// __glibcxx_assert(__m != memory_order_acquire);
+	// __glibcxx_assert(__m != memory_order_acq_rel);
+	// __glibcxx_assert(__m != memory_order_consume);
+
+	__atomic_store_n(&_M_p, __p, __m);
+      }
+
+      __pointer_type
+      load(memory_order __m = memory_order_seq_cst) const noexcept
+      {
+	// __glibcxx_assert(__m != memory_order_release);
+	// __glibcxx_assert(__m != memory_order_acq_rel);
+
+	return __atomic_load_n(&_M_p, __m);
+      }
+
+      __pointer_type
+      load(memory_order __m = memory_order_seq_cst) const volatile noexcept
+      {
+	// __glibcxx_assert(__m != memory_order_release);
+	// __glibcxx_assert(__m != memory_order_acq_rel);
+
+	return __atomic_load_n(&_M_p, __m);
+      }
+
+      __pointer_type
+      exchange(__pointer_type __p,
+	       memory_order __m = memory_order_seq_cst) noexcept
+      {
+	return __atomic_exchange_n(&_M_p, __p, __m);
+      }
+
+
+      __pointer_type
+      exchange(__pointer_type __p,
+	       memory_order __m = memory_order_seq_cst) volatile noexcept
+      {
+	return __atomic_exchange_n(&_M_p, __p, __m);
+      }
+
+      bool
+      compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
+			      memory_order __m1,
+			      memory_order __m2) noexcept
+      {
+	// __glibcxx_assert(__m2 != memory_order_release);
+	// __glibcxx_assert(__m2 != memory_order_acq_rel);
+	// __glibcxx_assert(__m2 <= __m1);
+
+	return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0, __m1, __m2);
+      }
+
+      bool
+      compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
+			      memory_order __m1,
+			      memory_order __m2) volatile noexcept
+      {
+	// __glibcxx_assert(__m2 != memory_order_release);
+	// __glibcxx_assert(__m2 != memory_order_acq_rel);
+	// __glibcxx_assert(__m2 <= __m1);
+
+	return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0, __m1, __m2);
+      }
+
+      __pointer_type
+      fetch_add(ptrdiff_t __d,
+		memory_order __m = memory_order_seq_cst) noexcept
+      { return __atomic_fetch_add(&_M_p, __d, __m); }
+
+      __pointer_type
+      fetch_add(ptrdiff_t __d,
+		memory_order __m = memory_order_seq_cst) volatile noexcept
+      { return __atomic_fetch_add(&_M_p, __d, __m); }
+
+      __pointer_type
+      fetch_sub(ptrdiff_t __d,
+		memory_order __m = memory_order_seq_cst) noexcept
+      { return __atomic_fetch_sub(&_M_p, __d, __m); }
+
+      __pointer_type
+      fetch_sub(ptrdiff_t __d,
+		memory_order __m = memory_order_seq_cst) volatile noexcept
+      { return __atomic_fetch_sub(&_M_p, __d, __m); }
+    };
+
+
+  /**
+   * @addtogroup atomics
+   * @{
+   */
+
+  /// atomic_bool
+  // NB: No operators or fetch-operations for this type.
+  struct atomic_bool
+  {
+  private:
+    __atomic_base<bool>	_M_base;
+
+  public:
+    atomic_bool() noexcept = default;
+    ~atomic_bool() noexcept = default;
+    atomic_bool(const atomic_bool&) = delete;
+    atomic_bool& operator=(const atomic_bool&) = delete;
+    atomic_bool& operator=(const atomic_bool&) volatile = delete;
+
+    constexpr atomic_bool(bool __i) noexcept : _M_base(__i) { }
+
+    bool
+    operator=(bool __i) noexcept
+    { return _M_base.operator=(__i); }
+
+    operator bool() const noexcept
+    { return _M_base.load(); }
+
+    operator bool() const volatile noexcept
+    { return _M_base.load(); }
+
+    bool
+    is_lock_free() const noexcept { return _M_base.is_lock_free(); }
+
+    bool
+    is_lock_free() const volatile noexcept { return _M_base.is_lock_free(); }
+
+    void
+    store(bool __i, memory_order __m = memory_order_seq_cst) noexcept
+    { _M_base.store(__i, __m); }
+
+    void
+    store(bool __i, memory_order __m = memory_order_seq_cst) volatile noexcept
+    { _M_base.store(__i, __m); }
+
+    bool
+    load(memory_order __m = memory_order_seq_cst) const noexcept
+    { return _M_base.load(__m); }
+
+    bool
+    load(memory_order __m = memory_order_seq_cst) const volatile noexcept
+    { return _M_base.load(__m); }
+
+    bool
+    exchange(bool __i, memory_order __m = memory_order_seq_cst) noexcept
+    { return _M_base.exchange(__i, __m); }
+
+    bool
+    exchange(bool __i,
+	     memory_order __m = memory_order_seq_cst) volatile noexcept
+    { return _M_base.exchange(__i, __m); }
+
+    bool
+    compare_exchange_weak(bool& __i1, bool __i2, memory_order __m1,
+			  memory_order __m2) noexcept
+    { return _M_base.compare_exchange_weak(__i1, __i2, __m1, __m2); }
+
+    bool
+    compare_exchange_weak(bool& __i1, bool __i2, memory_order __m1,
+			  memory_order __m2) volatile noexcept
+    { return _M_base.compare_exchange_weak(__i1, __i2, __m1, __m2); }
+
+    bool
+    compare_exchange_weak(bool& __i1, bool __i2,
+			  memory_order __m = memory_order_seq_cst) noexcept
+    { return _M_base.compare_exchange_weak(__i1, __i2, __m); }
+
+    bool
+    compare_exchange_weak(bool& __i1, bool __i2,
+		     memory_order __m = memory_order_seq_cst) volatile noexcept
+    { return _M_base.compare_exchange_weak(__i1, __i2, __m); }
+
+    bool
+    compare_exchange_strong(bool& __i1, bool __i2, memory_order __m1,
+			    memory_order __m2) noexcept
+    { return _M_base.compare_exchange_strong(__i1, __i2, __m1, __m2); }
+
+    bool
+    compare_exchange_strong(bool& __i1, bool __i2, memory_order __m1,
+			    memory_order __m2) volatile noexcept
+    { return _M_base.compare_exchange_strong(__i1, __i2, __m1, __m2); }
+
+    bool
+    compare_exchange_strong(bool& __i1, bool __i2,
+			    memory_order __m = memory_order_seq_cst) noexcept
+    { return _M_base.compare_exchange_strong(__i1, __i2, __m); }
+
+    bool
+    compare_exchange_strong(bool& __i1, bool __i2,
+		    memory_order __m = memory_order_seq_cst) volatile noexcept
+    { return _M_base.compare_exchange_strong(__i1, __i2, __m); }
+  };
+
+
+  /// atomic
+  /// 29.4.3, Generic atomic type, primary class template.
+  template<typename _Tp>
+    struct atomic
+    {
+    private:
+      _Tp _M_i;
+
+    public:
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(_Tp __i) noexcept : _M_i(__i) { }
+
+      operator _Tp() const noexcept
+      { return load(); }
+
+      operator _Tp() const volatile noexcept
+      { return load(); }
+
+      _Tp
+      operator=(_Tp __i) noexcept 
+      { store(__i); return __i; }
+
+      _Tp
+      operator=(_Tp __i) volatile noexcept 
+      { store(__i); return __i; }
+
+      bool
+      is_lock_free() const noexcept
+      { return __atomic_is_lock_free(sizeof(_M_i), &_M_i); }
+
+      bool
+      is_lock_free() const volatile noexcept
+      { return __atomic_is_lock_free(sizeof(_M_i), &_M_i); }
+
+      void
+      store(_Tp __i, memory_order _m = memory_order_seq_cst) noexcept
+      { __atomic_store(&_M_i, &__i, _m); }
+
+      void
+      store(_Tp __i, memory_order _m = memory_order_seq_cst) volatile noexcept
+      { __atomic_store(&_M_i, &__i, _m); }
+
+      _Tp
+      load(memory_order _m = memory_order_seq_cst) const noexcept
+      { 
+        _Tp tmp;
+	__atomic_load(&_M_i, &tmp, _m); 
+	return tmp;
+      }
+
+      _Tp
+      load(memory_order _m = memory_order_seq_cst) const volatile noexcept
+      { 
+        _Tp tmp;
+	__atomic_load(&_M_i, &tmp, _m); 
+	return tmp;
+      }
+
+      _Tp
+      exchange(_Tp __i, memory_order _m = memory_order_seq_cst) noexcept
+      { 
+        _Tp tmp;
+	__atomic_exchange(&_M_i, &__i, &tmp, _m); 
+	return tmp;
+      }
+
+      _Tp
+      exchange(_Tp __i, 
+	       memory_order _m = memory_order_seq_cst) volatile noexcept
+      { 
+        _Tp tmp;
+	__atomic_exchange(&_M_i, &__i, &tmp, _m); 
+	return tmp;
+      }
+
+      bool
+      compare_exchange_weak(_Tp& __e, _Tp __i, memory_order __s, 
+			    memory_order __f) noexcept
+      {
+	return __atomic_compare_exchange(&_M_i, &__e, &__i, true, __s, __f); 
+      }
+
+      bool
+      compare_exchange_weak(_Tp& __e, _Tp __i, memory_order __s, 
+			    memory_order __f) volatile noexcept
+      {
+	return __atomic_compare_exchange(&_M_i, &__e, &__i, true, __s, __f); 
+      }
+
+      bool
+      compare_exchange_weak(_Tp& __e, _Tp __i,
+			    memory_order __m = memory_order_seq_cst) noexcept
+      { return compare_exchange_weak(__e, __i, __m, __m); }
+
+      bool
+      compare_exchange_weak(_Tp& __e, _Tp __i,
+		     memory_order __m = memory_order_seq_cst) volatile noexcept
+      { return compare_exchange_weak(__e, __i, __m, __m); }
+
+      bool
+      compare_exchange_strong(_Tp& __e, _Tp __i, memory_order __s, 
+			      memory_order __f) noexcept
+      {
+	return __atomic_compare_exchange(&_M_i, &__e, &__i, false, __s, __f); 
+      }
+
+      bool
+      compare_exchange_strong(_Tp& __e, _Tp __i, memory_order __s, 
+			      memory_order __f) volatile noexcept
+      {
+	return __atomic_compare_exchange(&_M_i, &__e, &__i, false, __s, __f); 
+      }
+
+      bool
+      compare_exchange_strong(_Tp& __e, _Tp __i,
+			       memory_order __m = memory_order_seq_cst) noexcept
+      { return compare_exchange_strong(__e, __i, __m, __m); }
+
+      bool
+      compare_exchange_strong(_Tp& __e, _Tp __i,
+		     memory_order __m = memory_order_seq_cst) volatile noexcept
+      { return compare_exchange_strong(__e, __i, __m, __m); }
+    };
+
+
+  /// Partial specialization for pointer types.
+  template<typename _Tp>
+    struct atomic<_Tp*>
+    {
+      typedef _Tp* 			__pointer_type;
+      typedef __atomic_base<_Tp*>	__base_type;
+      __base_type			_M_b;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__pointer_type __p) noexcept : _M_b(__p) { }
+
+      operator __pointer_type() const noexcept
+      { return __pointer_type(_M_b); }
+
+      operator __pointer_type() const volatile noexcept
+      { return __pointer_type(_M_b); }
+
+      __pointer_type
+      operator=(__pointer_type __p) noexcept
+      { return _M_b.operator=(__p); }
+
+      __pointer_type
+      operator=(__pointer_type __p) volatile noexcept
+      { return _M_b.operator=(__p); }
+
+      __pointer_type
+      operator++(int) noexcept
+      { return _M_b++; }
+
+      __pointer_type
+      operator++(int) volatile noexcept
+      { return _M_b++; }
+
+      __pointer_type
+      operator--(int) noexcept
+      { return _M_b--; }
+
+      __pointer_type
+      operator--(int) volatile noexcept
+      { return _M_b--; }
+
+      __pointer_type
+      operator++() noexcept
+      { return ++_M_b; }
+
+      __pointer_type
+      operator++() volatile noexcept
+      { return ++_M_b; }
+
+      __pointer_type
+      operator--() noexcept
+      { return --_M_b; }
+
+      __pointer_type
+      operator--() volatile noexcept
+      { return --_M_b; }
+
+      __pointer_type
+      operator+=(ptrdiff_t __d) noexcept
+      { return _M_b.operator+=(__d); }
+
+      __pointer_type
+      operator+=(ptrdiff_t __d) volatile noexcept
+      { return _M_b.operator+=(__d); }
+
+      __pointer_type
+      operator-=(ptrdiff_t __d) noexcept
+      { return _M_b.operator-=(__d); }
+
+      __pointer_type
+      operator-=(ptrdiff_t __d) volatile noexcept
+      { return _M_b.operator-=(__d); }
+
+      bool
+      is_lock_free() const noexcept
+      { return _M_b.is_lock_free(); }
+
+      bool
+      is_lock_free() const volatile noexcept
+      { return _M_b.is_lock_free(); }
+
+      void
+      store(__pointer_type __p,
+	    memory_order __m = memory_order_seq_cst) noexcept
+      { return _M_b.store(__p, __m); }
+
+      void
+      store(__pointer_type __p,
+	    memory_order __m = memory_order_seq_cst) volatile noexcept
+      { return _M_b.store(__p, __m); }
+
+      __pointer_type
+      load(memory_order __m = memory_order_seq_cst) const noexcept
+      { return _M_b.load(__m); }
+
+      __pointer_type
+      load(memory_order __m = memory_order_seq_cst) const volatile noexcept
+      { return _M_b.load(__m); }
+
+      __pointer_type
+      exchange(__pointer_type __p,
+	       memory_order __m = memory_order_seq_cst) noexcept
+      { return _M_b.exchange(__p, __m); }
+
+      __pointer_type
+      exchange(__pointer_type __p,
+	       memory_order __m = memory_order_seq_cst) volatile noexcept
+      { return _M_b.exchange(__p, __m); }
+
+      bool
+      compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
+			    memory_order __m1, memory_order __m2) noexcept
+      { return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }
+
+      bool
+      compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
+			    memory_order __m1,
+			    memory_order __m2) volatile noexcept
+      { return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }
+
+      bool
+      compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
+			    memory_order __m = memory_order_seq_cst) noexcept
+      {
+	return compare_exchange_weak(__p1, __p2, __m,
+				     __calculate_memory_order(__m));
+      }
+
+      bool
+      compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
+		    memory_order __m = memory_order_seq_cst) volatile noexcept
+      {
+	return compare_exchange_weak(__p1, __p2, __m,
+				     __calculate_memory_order(__m));
+      }
+
+      bool
+      compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
+			      memory_order __m1, memory_order __m2) noexcept
+      { return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }
+
+      bool
+      compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
+			      memory_order __m1,
+			      memory_order __m2) volatile noexcept
+      { return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }
+
+      bool
+      compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
+			      memory_order __m = memory_order_seq_cst) noexcept
+      {
+	return _M_b.compare_exchange_strong(__p1, __p2, __m,
+					    __calculate_memory_order(__m));
+      }
+
+      bool
+      compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
+		    memory_order __m = memory_order_seq_cst) volatile noexcept
+      {
+	return _M_b.compare_exchange_strong(__p1, __p2, __m,
+					    __calculate_memory_order(__m));
+      }
+
+      __pointer_type
+      fetch_add(ptrdiff_t __d,
+		memory_order __m = memory_order_seq_cst) noexcept
+      { return _M_b.fetch_add(__d, __m); }
+
+      __pointer_type
+      fetch_add(ptrdiff_t __d,
+		memory_order __m = memory_order_seq_cst) volatile noexcept
+      { return _M_b.fetch_add(__d, __m); }
+
+      __pointer_type
+      fetch_sub(ptrdiff_t __d,
+		memory_order __m = memory_order_seq_cst) noexcept
+      { return _M_b.fetch_sub(__d, __m); }
+
+      __pointer_type
+      fetch_sub(ptrdiff_t __d,
+		memory_order __m = memory_order_seq_cst) volatile noexcept
+      { return _M_b.fetch_sub(__d, __m); }
+    };
+
+
+  /// Explicit specialization for bool.
+  template<>
+    struct atomic<bool> : public atomic_bool
+    {
+      typedef bool 			__integral_type;
+      typedef atomic_bool 		__base_type;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for char.
+  template<>
+    struct atomic<char> : public atomic_char
+    {
+      typedef char 			__integral_type;
+      typedef atomic_char 		__base_type;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for signed char.
+  template<>
+    struct atomic<signed char> : public atomic_schar
+    {
+      typedef signed char 		__integral_type;
+      typedef atomic_schar 		__base_type;
+
+      atomic() noexcept= default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for unsigned char.
+  template<>
+    struct atomic<unsigned char> : public atomic_uchar
+    {
+      typedef unsigned char 		__integral_type;
+      typedef atomic_uchar 		__base_type;
+
+      atomic() noexcept= default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for short.
+  template<>
+    struct atomic<short> : public atomic_short
+    {
+      typedef short 			__integral_type;
+      typedef atomic_short 		__base_type;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for unsigned short.
+  template<>
+    struct atomic<unsigned short> : public atomic_ushort
+    {
+      typedef unsigned short 	      	__integral_type;
+      typedef atomic_ushort 		__base_type;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for int.
+  template<>
+    struct atomic<int> : atomic_int
+    {
+      typedef int 			__integral_type;
+      typedef atomic_int 		__base_type;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for unsigned int.
+  template<>
+    struct atomic<unsigned int> : public atomic_uint
+    {
+      typedef unsigned int		__integral_type;
+      typedef atomic_uint 		__base_type;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for long.
+  template<>
+    struct atomic<long> : public atomic_long
+    {
+      typedef long 			__integral_type;
+      typedef atomic_long 		__base_type;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for unsigned long.
+  template<>
+    struct atomic<unsigned long> : public atomic_ulong
+    {
+      typedef unsigned long 		__integral_type;
+      typedef atomic_ulong 		__base_type;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for long long.
+  template<>
+    struct atomic<long long> : public atomic_llong
+    {
+      typedef long long 		__integral_type;
+      typedef atomic_llong 		__base_type;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for unsigned long long.
+  template<>
+    struct atomic<unsigned long long> : public atomic_ullong
+    {
+      typedef unsigned long long       	__integral_type;
+      typedef atomic_ullong 		__base_type;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for wchar_t.
+  template<>
+    struct atomic<wchar_t> : public atomic_wchar_t
+    {
+      typedef wchar_t 			__integral_type;
+      typedef atomic_wchar_t 		__base_type;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for char16_t.
+  template<>
+    struct atomic<char16_t> : public atomic_char16_t
+    {
+      typedef char16_t 			__integral_type;
+      typedef atomic_char16_t 		__base_type;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+  /// Explicit specialization for char32_t.
+  template<>
+    struct atomic<char32_t> : public atomic_char32_t
+    {
+      typedef char32_t 			__integral_type;
+      typedef atomic_char32_t 		__base_type;
+
+      atomic() noexcept = default;
+      ~atomic() noexcept = default;
+      atomic(const atomic&) = delete;
+      atomic& operator=(const atomic&) = delete;
+      atomic& operator=(const atomic&) volatile = delete;
+
+      constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
+
+      using __base_type::operator __integral_type;
+      using __base_type::operator=;
+    };
+
+
+  // Function definitions, atomic_flag operations.
+  inline bool
+  atomic_flag_test_and_set_explicit(atomic_flag* __a,
+				    memory_order __m) noexcept
+  { return __a->test_and_set(__m); }
+
+  inline bool
+  atomic_flag_test_and_set_explicit(volatile atomic_flag* __a,
+				    memory_order __m) noexcept
+  { return __a->test_and_set(__m); }
+
+  inline void
+  atomic_flag_clear_explicit(atomic_flag* __a, memory_order __m) noexcept
+  { __a->clear(__m); }
+
+  inline void
+  atomic_flag_clear_explicit(volatile atomic_flag* __a,
+			     memory_order __m) noexcept
+  { __a->clear(__m); }
+
+  inline bool
+  atomic_flag_test_and_set(atomic_flag* __a) noexcept
+  { return atomic_flag_test_and_set_explicit(__a, memory_order_seq_cst); }
+
+  inline bool
+  atomic_flag_test_and_set(volatile atomic_flag* __a) noexcept
+  { return atomic_flag_test_and_set_explicit(__a, memory_order_seq_cst); }
+
+  inline void
+  atomic_flag_clear(atomic_flag* __a) noexcept
+  { atomic_flag_clear_explicit(__a, memory_order_seq_cst); }
+
+  inline void
+  atomic_flag_clear(volatile atomic_flag* __a) noexcept
+  { atomic_flag_clear_explicit(__a, memory_order_seq_cst); }
+
+
+  // Function templates generally applicable to atomic types.
+  template<typename _ITp>
+    inline bool
+    atomic_is_lock_free(const atomic<_ITp>* __a) noexcept
+    { return __a->is_lock_free(); }
+
+  template<typename _ITp>
+    inline bool
+    atomic_is_lock_free(const volatile atomic<_ITp>* __a) noexcept
+    { return __a->is_lock_free(); }
+
+  template<typename _ITp>
+    inline void
+    atomic_init(atomic<_ITp>* __a, _ITp __i) noexcept;
+
+  template<typename _ITp>
+    inline void
+    atomic_init(volatile atomic<_ITp>* __a, _ITp __i) noexcept;
+
+  template<typename _ITp>
+    inline void
+    atomic_store_explicit(atomic<_ITp>* __a, _ITp __i,
+			  memory_order __m) noexcept
+    { __a->store(__i, __m); }
+
+  template<typename _ITp>
+    inline void
+    atomic_store_explicit(volatile atomic<_ITp>* __a, _ITp __i,
+			  memory_order __m) noexcept
+    { __a->store(__i, __m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_load_explicit(const atomic<_ITp>* __a, memory_order __m) noexcept
+    { return __a->load(__m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_load_explicit(const volatile atomic<_ITp>* __a,
+			 memory_order __m) noexcept
+    { return __a->load(__m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_exchange_explicit(atomic<_ITp>* __a, _ITp __i,
+			     memory_order __m) noexcept
+    { return __a->exchange(__i, __m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_exchange_explicit(volatile atomic<_ITp>* __a, _ITp __i,
+			     memory_order __m) noexcept
+    { return __a->exchange(__i, __m); }
+
+  template<typename _ITp>
+    inline bool
+    atomic_compare_exchange_weak_explicit(atomic<_ITp>* __a,
+					  _ITp* __i1, _ITp __i2,
+					  memory_order __m1,
+					  memory_order __m2) noexcept
+    { return __a->compare_exchange_weak(*__i1, __i2, __m1, __m2); }
+
+  template<typename _ITp>
+    inline bool
+    atomic_compare_exchange_weak_explicit(volatile atomic<_ITp>* __a,
+					  _ITp* __i1, _ITp __i2,
+					  memory_order __m1,
+					  memory_order __m2) noexcept
+    { return __a->compare_exchange_weak(*__i1, __i2, __m1, __m2); }
+
+  template<typename _ITp>
+    inline bool
+    atomic_compare_exchange_strong_explicit(atomic<_ITp>* __a,
+					    _ITp* __i1, _ITp __i2,
+					    memory_order __m1,
+					    memory_order __m2) noexcept
+    { return __a->compare_exchange_strong(*__i1, __i2, __m1, __m2); }
+
+  template<typename _ITp>
+    inline bool
+    atomic_compare_exchange_strong_explicit(volatile atomic<_ITp>* __a,
+					    _ITp* __i1, _ITp __i2,
+					    memory_order __m1,
+					    memory_order __m2) noexcept
+    { return __a->compare_exchange_strong(*__i1, __i2, __m1, __m2); }
+
+
+  template<typename _ITp>
+    inline void
+    atomic_store(atomic<_ITp>* __a, _ITp __i) noexcept
+    { atomic_store_explicit(__a, __i, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline void
+    atomic_store(volatile atomic<_ITp>* __a, _ITp __i) noexcept
+    { atomic_store_explicit(__a, __i, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_load(const atomic<_ITp>* __a) noexcept
+    { return atomic_load_explicit(__a, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_load(const volatile atomic<_ITp>* __a) noexcept
+    { return atomic_load_explicit(__a, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_exchange(atomic<_ITp>* __a, _ITp __i) noexcept
+    { return atomic_exchange_explicit(__a, __i, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_exchange(volatile atomic<_ITp>* __a, _ITp __i) noexcept
+    { return atomic_exchange_explicit(__a, __i, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline bool
+    atomic_compare_exchange_weak(atomic<_ITp>* __a,
+				 _ITp* __i1, _ITp __i2) noexcept
+    {
+      return atomic_compare_exchange_weak_explicit(__a, __i1, __i2,
+						   memory_order_seq_cst,
+						   memory_order_seq_cst);
+    }
+
+  template<typename _ITp>
+    inline bool
+    atomic_compare_exchange_weak(volatile atomic<_ITp>* __a,
+				 _ITp* __i1, _ITp __i2) noexcept
+    {
+      return atomic_compare_exchange_weak_explicit(__a, __i1, __i2,
+						   memory_order_seq_cst,
+						   memory_order_seq_cst);
+    }
+
+  template<typename _ITp>
+    inline bool
+    atomic_compare_exchange_strong(atomic<_ITp>* __a,
+				   _ITp* __i1, _ITp __i2) noexcept
+    {
+      return atomic_compare_exchange_strong_explicit(__a, __i1, __i2,
+						     memory_order_seq_cst,
+						     memory_order_seq_cst);
+    }
+
+  template<typename _ITp>
+    inline bool
+    atomic_compare_exchange_strong(volatile atomic<_ITp>* __a,
+				   _ITp* __i1, _ITp __i2) noexcept
+    {
+      return atomic_compare_exchange_strong_explicit(__a, __i1, __i2,
+						     memory_order_seq_cst,
+						     memory_order_seq_cst);
+    }
+
+  // Function templates for atomic_integral operations only, using
+  // __atomic_base. Template argument should be constricted to
+  // intergral types as specified in the standard, excluding address
+  // types.
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_add_explicit(__atomic_base<_ITp>* __a, _ITp __i,
+			      memory_order __m) noexcept
+    { return __a->fetch_add(__i, __m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_add_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
+			      memory_order __m) noexcept
+    { return __a->fetch_add(__i, __m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_sub_explicit(__atomic_base<_ITp>* __a, _ITp __i,
+			      memory_order __m) noexcept
+    { return __a->fetch_sub(__i, __m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_sub_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
+			      memory_order __m) noexcept
+    { return __a->fetch_sub(__i, __m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_and_explicit(__atomic_base<_ITp>* __a, _ITp __i,
+			      memory_order __m) noexcept
+    { return __a->fetch_and(__i, __m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_and_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
+			      memory_order __m) noexcept
+    { return __a->fetch_and(__i, __m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_or_explicit(__atomic_base<_ITp>* __a, _ITp __i,
+			     memory_order __m) noexcept
+    { return __a->fetch_or(__i, __m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_or_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
+			     memory_order __m) noexcept
+    { return __a->fetch_or(__i, __m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_xor_explicit(__atomic_base<_ITp>* __a, _ITp __i,
+			      memory_order __m) noexcept
+    { return __a->fetch_xor(__i, __m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_xor_explicit(volatile __atomic_base<_ITp>* __a, _ITp __i,
+			      memory_order __m) noexcept
+    { return __a->fetch_xor(__i, __m); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_add(__atomic_base<_ITp>* __a, _ITp __i) noexcept
+    { return atomic_fetch_add_explicit(__a, __i, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_add(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
+    { return atomic_fetch_add_explicit(__a, __i, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_sub(__atomic_base<_ITp>* __a, _ITp __i) noexcept
+    { return atomic_fetch_sub_explicit(__a, __i, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_sub(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
+    { return atomic_fetch_sub_explicit(__a, __i, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_and(__atomic_base<_ITp>* __a, _ITp __i) noexcept
+    { return atomic_fetch_and_explicit(__a, __i, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_and(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
+    { return atomic_fetch_and_explicit(__a, __i, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_or(__atomic_base<_ITp>* __a, _ITp __i) noexcept
+    { return atomic_fetch_or_explicit(__a, __i, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_or(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
+    { return atomic_fetch_or_explicit(__a, __i, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_xor(__atomic_base<_ITp>* __a, _ITp __i) noexcept
+    { return atomic_fetch_xor_explicit(__a, __i, memory_order_seq_cst); }
+
+  template<typename _ITp>
+    inline _ITp
+    atomic_fetch_xor(volatile __atomic_base<_ITp>* __a, _ITp __i) noexcept
+    { return atomic_fetch_xor_explicit(__a, __i, memory_order_seq_cst); }
+
+
+  // Partial specializations for pointers.
+  template<typename _ITp>
+    inline _ITp*
+    atomic_fetch_add_explicit(atomic<_ITp*>* __a, ptrdiff_t __d,
+			      memory_order __m) noexcept
+    { return __a->fetch_add(__d, __m); }
+
+  template<typename _ITp>
+    inline _ITp*
+    atomic_fetch_add_explicit(volatile atomic<_ITp*>* __a, ptrdiff_t __d,
+			      memory_order __m) noexcept
+    { return __a->fetch_add(__d, __m); }
+
+  template<typename _ITp>
+    inline _ITp*
+    atomic_fetch_add(volatile atomic<_ITp*>* __a, ptrdiff_t __d) noexcept
+    { return __a->fetch_add(__d); }
+
+  template<typename _ITp>
+    inline _ITp*
+    atomic_fetch_add(atomic<_ITp*>* __a, ptrdiff_t __d) noexcept
+    { return __a->fetch_add(__d); }
+
+  template<typename _ITp>
+    inline _ITp*
+    atomic_fetch_sub_explicit(volatile atomic<_ITp*>* __a,
+			      ptrdiff_t __d, memory_order __m) noexcept
+    { return __a->fetch_sub(__d, __m); }
+
+  template<typename _ITp>
+    inline _ITp*
+    atomic_fetch_sub_explicit(atomic<_ITp*>* __a, ptrdiff_t __d,
+			      memory_order __m) noexcept
+    { return __a->fetch_sub(__d, __m); }
+
+  template<typename _ITp>
+    inline _ITp*
+    atomic_fetch_sub(volatile atomic<_ITp*>* __a, ptrdiff_t __d) noexcept
+    { return __a->fetch_sub(__d); }
+
+  template<typename _ITp>
+    inline _ITp*
+    atomic_fetch_sub(atomic<_ITp*>* __a, ptrdiff_t __d) noexcept
+    { return __a->fetch_sub(__d); }
+  // @} group atomics
+
+// _GLIBCXX_END_NAMESPACE_VERSION
+} // namespace
+
+#endif
diff --git a/libitm/method-gl.cc b/libitm/method-gl.cc
index 60c2662..81045d3 100644
--- a/libitm/method-gl.cc
+++ b/libitm/method-gl.cc
@@ -41,10 +41,11 @@ struct gl_mg : public method_group
   static gtm_word clear_locked(gtm_word l) { return l & ~LOCK_BIT; }
 
   // The global ownership record.
-  gtm_word orec;
+  atomic<gtm_word> orec;
+
   virtual void init()
   {
-    orec = 0;
+    orec.store(0, memory_order_relaxed);
   }
   virtual void fini() { }
 };
@@ -84,28 +85,25 @@ protected:
   static void pre_write(const void *addr, size_t len)
   {
     gtm_thread *tx = gtm_thr();
-    if (unlikely(!gl_mg::is_locked(tx->shared_state)))
+    gtm_word v = tx->shared_state.load(memory_order_acquire);
+    if (unlikely(!gl_mg::is_locked(v)))
       {
 	// Check for and handle version number overflow.
-	if (unlikely(tx->shared_state >= gl_mg::VERSION_MAX))
+	if (unlikely(v >= gl_mg::VERSION_MAX))
 	  tx->restart(RESTART_INIT_METHOD_GROUP);
 
 	// CAS global orec from our snapshot time to the locked state.
 	// This validates that we have a consistent snapshot, which is also
 	// for making privatization safety work (see the class' comments).
-	gtm_word now = o_gl_mg.orec;
-	if (now != tx->shared_state)
+	gtm_word now = o_gl_mg.orec.load(memory_order_relaxed);
+	if (now != v)
 	  tx->restart(RESTART_VALIDATE_WRITE);
-	if (__sync_val_compare_and_swap(&o_gl_mg.orec, now,
-	    gl_mg::set_locked(now)) != now)
+	if (!o_gl_mg.orec.compare_exchange_strong (now, gl_mg::set_locked(now),
+						   memory_order_acquire))
 	  tx->restart(RESTART_LOCKED_WRITE);
 
-	// Set shared_state to new value. The CAS is a full barrier, so the
-	// acquisition of the global orec is visible before this store here,
-	// and the store will not be visible before earlier data loads, which
-	// is required to correctly ensure privatization safety (see
-	// begin_and_restart() and release_orec() for further comments).
-	tx->shared_state = gl_mg::set_locked(now);
+	// Set shared_state to new value.
+	tx->shared_state.store(gl_mg::set_locked(now), memory_order_release);
       }
 
     // TODO Ensure that this gets inlined: Use internal log interface and LTO.
@@ -115,11 +113,12 @@ protected:
   static void validate()
   {
     // Check that snapshot is consistent. The barrier ensures that this
-    // happens after previous data loads.
-    atomic_read_barrier();
+    // happens after previous data loads.  Recall that load cannot itself
+    // have memory_order_release.
     gtm_thread *tx = gtm_thr();
-    gtm_word l = o_gl_mg.orec;
-    if (l != tx->shared_state)
+    atomic_thread_fence(memory_order_release);
+    gtm_word l = o_gl_mg.orec.load(memory_order_relaxed);
+    if (l != tx->shared_state.load(memory_order_relaxed))
       tx->restart(RESTART_VALIDATE_READ);
   }
 
@@ -180,17 +179,18 @@ public:
 
     // Spin until global orec is not locked.
     // TODO This is not necessary if there are no pure loads (check txn props).
-    gtm_word v;
     unsigned i = 0;
-    while (gl_mg::is_locked(v = o_gl_mg.orec))
+    gtm_word v;
+    while (1)
       {
+        v = o_gl_mg.orec.load(memory_order_acquire);
+        if (!gl_mg::is_locked(v))
+	  break;
 	// TODO need method-specific max spin count
-	if (++i > gtm_spin_count_var) return RESTART_VALIDATE_READ;
+	if (++i > gtm_spin_count_var)
+	  return RESTART_VALIDATE_READ;
 	cpu_relax();
       }
-    // This barrier ensures that we have read the global orec before later
-    // data loads.
-    atomic_read_barrier();
 
     // Everything is okay, we have a snapshot time.
     // We don't need to enforce any ordering for the following store. There
@@ -202,14 +202,14 @@ public:
     // marking the transaction as active, and restarts enforce immediate
     // visibility of a smaller or equal value with a barrier (see
     // release_orec()).
-    tx->shared_state = v;
+    tx->shared_state.store(v, memory_order_relaxed);
     return NO_RESTART;
   }
 
   virtual bool trycommit(gtm_word& priv_time)
   {
     gtm_thread* tx = gtm_thr();
-    gtm_word v = tx->shared_state;
+    gtm_word v = tx->shared_state.load(memory_order_acquire);
 
     // Special case: If shared_state is ~0, then we have acquired the
     // serial lock (tx->state is not updated yet). In this case, the previous
@@ -218,7 +218,7 @@ public:
     // anymore. In particular, if it is locked, then we are an update
     // transaction, which is all we care about for commit.
     if (v == ~(typeof v)0)
-      v = o_gl_mg.orec;
+      v = o_gl_mg.orec.load(memory_order_relaxed);
 
     // Release the orec but do not reset shared_state, which will be modified
     // by the serial lock right after our commit anyway. Also, resetting
@@ -227,10 +227,8 @@ public:
     if (gl_mg::is_locked(v))
       {
 	// Release the global orec, increasing its version number / timestamp.
-	// TODO replace with C++0x-style atomics (a release in this case)
-	atomic_write_barrier();
 	v = gl_mg::clear_locked(v) + 1;
-	o_gl_mg.orec = v;
+	o_gl_mg.orec.store(v, memory_order_release);
 
 	// Need to ensure privatization safety. Every other transaction must
 	// have a snapshot time that is at least as high as our commit time
@@ -247,15 +245,16 @@ public:
       return;
 
     gtm_thread *tx = gtm_thr();
-    gtm_word v = tx->shared_state;
+    gtm_word v = tx->shared_state.load(memory_order_acquire);
     // Special case: If shared_state is ~0, then we have acquired the
     // serial lock (tx->state is not updated yet). In this case, the previous
     // value isn't available anymore, so grab it from the global lock, which
     // must have a meaningful value because no other transactions are active
     // anymore. In particular, if it is locked, then we are an update
     // transaction, which is all we care about for rollback.
-    if (v == ~(typeof v)0)
-      v = o_gl_mg.orec;
+    bool is_serial = v == ~(typeof v)0;
+    if (is_serial)
+      v = o_gl_mg.orec.load(memory_order_relaxed);
 
     // Release lock and increment version number to prevent dirty reads.
     // Also reset shared state here, so that begin_or_restart() can expect a
@@ -263,16 +262,14 @@ public:
     if (gl_mg::is_locked(v))
       {
 	// Release the global orec, increasing its version number / timestamp.
-	// TODO replace with C++0x-style atomics (a release in this case)
-	atomic_write_barrier();
 	v = gl_mg::clear_locked(v) + 1;
-	o_gl_mg.orec = v;
+	o_gl_mg.orec.store(v, memory_order_release);
 
 	// Also reset the timestamp published via shared_state.
 	// Special case: Only do this if we are not a serial transaction
 	// because otherwise, we would interfere with the serial lock.
-	if (tx->shared_state != ~(typeof tx->shared_state)0)
-	  tx->shared_state = v;
+	if (!is_serial)
+	  tx->shared_state.store(v, memory_order_relaxed);
 
 	// We need a store-load barrier after this store to prevent it
 	// from becoming visible after later data loads because the
@@ -280,7 +277,7 @@ public:
 	// snapshot time (the lock bit had been set), which could break
 	// privatization safety. We do not need a barrier before this
 	// store (see pre_write() for an explanation).
-	__sync_synchronize();
+	atomic_thread_fence(memory_order_acq_rel);
       }
 
   }
diff --git a/libitm/stmlock.h b/libitm/stmlock.h
index f115a41..a338258 100644
--- a/libitm/stmlock.h
+++ b/libitm/stmlock.h
@@ -92,24 +92,23 @@ gtm_get_stmlock (const gtm_cacheline *addr)
 }
 
 /* The current global version number.  */
-extern gtm_version gtm_clock;
+extern atomic<gtm_version> gtm_clock;
 
 static inline gtm_version
 gtm_get_clock (void)
 {
-  gtm_version r;
-
-  __sync_synchronize ();
-  r = gtm_clock;
-  atomic_read_barrier ();
-
-  return r;
+  atomic_thread_fence(memory_order_release);
+  return gtm_clock.load(memory_order_acquire);
 }
 
 static inline gtm_version
 gtm_inc_clock (void)
 {
-  gtm_version r = __sync_add_and_fetch (&gtm_clock, 1);
+  /* ??? Here we have a choice, the pre-inc operator mapping to
+     __atomic_add_fetch with memory_order_seq_cst, or fetch_add
+     with memory_order_acq_rel plus another separate increment.
+     We really ought to recognize and optimize fetch_op(x) op x... */
+  gtm_version r = ++gtm_clock;
 
   /* ??? Ought to handle wraparound for 32-bit.  */
   if (sizeof(r) < 8 && r > GTM_VERSION_MAX)