Merge from transactional-memory branch.

From-SVN: r181154

7 files changed, 665 insertions, 0 deletions

diff --git a/libitm/config/linux/alpha/futex_bits.h b/libitm/config/linux/alpha/futex_bits.h
new file mode 100644
index 0000000..997bf0b
--- /dev/null
+++ b/libitm/config/linux/alpha/futex_bits.h
@@ -0,0 +1,56 @@
+/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
+   Contributed by Richard Henderson <rth@redhat.com>.
+
+   This file is part of the GNU Transactional Memory Library (libitm).
+
+   Libitm is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Provide target-specific access to the futex system call.  */
+
+#ifndef SYS_futex
+#define SYS_futex               394
+#endif
+
+static inline long
+sys_futex0 (int *addr, long op, long val)
+{
+  register long sc_0 __asm__("$0");
+  register long sc_16 __asm__("$16");
+  register long sc_17 __asm__("$17");
+  register long sc_18 __asm__("$18");
+  register long sc_19 __asm__("$19");
+  long res;
+
+  sc_0 = SYS_futex;
+  sc_16 = (long) addr;
+  sc_17 = op;
+  sc_18 = val;
+  sc_19 = 0;
+  __asm volatile ("callsys"
+		  : "=r" (sc_0), "=r"(sc_19)
+		  : "0"(sc_0), "r" (sc_16), "r"(sc_17), "r"(sc_18), "1"(sc_19)
+		  : "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8",
+		    "$22", "$23", "$24", "$25", "$27", "$28", "memory");
+
+  res = sc_0;
+  if (__builtin_expect (sc_19, 0))
+    res = -res;
+  return res;
+}
diff --git a/libitm/config/linux/futex.cc b/libitm/config/linux/futex.cc
new file mode 100644
index 0000000..45c9db6
--- /dev/null
+++ b/libitm/config/linux/futex.cc
@@ -0,0 +1,82 @@
+/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
+   Contributed by Richard Henderson <rth@redhat.com>.
+
+   This file is part of the GNU Transactional Memory Library (libitm).
+
+   Libitm is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Provide access to the futex system call.  */
+
+#include "libitm_i.h"
+#include "futex.h"
+#include <errno.h>
+
+namespace GTM HIDDEN {
+
+#define FUTEX_WAIT		0
+#define FUTEX_WAKE		1
+#define FUTEX_PRIVATE_FLAG	128L
+
+
+static long int gtm_futex_wait = FUTEX_WAIT | FUTEX_PRIVATE_FLAG;
+static long int gtm_futex_wake = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
+
+
+void
+futex_wait (int *addr, int val)
+{
+  long res;
+
+  res = sys_futex0 (addr, gtm_futex_wait, val);
+  if (__builtin_expect (res == -ENOSYS, 0))
+    {
+      gtm_futex_wait = FUTEX_WAIT;
+      gtm_futex_wake = FUTEX_WAKE;
+      res = sys_futex0 (addr, FUTEX_WAIT, val);
+    }
+  if (__builtin_expect (res < 0, 0))
+    {
+      if (res == -EWOULDBLOCK || res == -ETIMEDOUT)
+	;
+      else if (res == -EFAULT)
+	GTM_fatal ("futex failed (EFAULT %p)", addr);
+      else
+	GTM_fatal ("futex failed (%s)", strerror(-res));
+    }
+}
+
+
+long
+futex_wake (int *addr, int count)
+{
+  long res = sys_futex0 (addr, gtm_futex_wake, count);
+  if (__builtin_expect (res == -ENOSYS, 0))
+    {
+      gtm_futex_wait = FUTEX_WAIT;
+      gtm_futex_wake = FUTEX_WAKE;
+      res = sys_futex0 (addr, FUTEX_WAKE, count);
+    }
+  if (__builtin_expect (res < 0, 0))
+    GTM_fatal ("futex failed (%s)", strerror(-res));
+  else
+    return res;
+}
+
+} // namespace GTM
diff --git a/libitm/config/linux/futex.h b/libitm/config/linux/futex.h
new file mode 100644
index 0000000..326c0f5
--- /dev/null
+++ b/libitm/config/linux/futex.h
@@ -0,0 +1,39 @@
+/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
+   Contributed by Richard Henderson <rth@redhat.com>.
+
+   This file is part of the GNU Transactional Memory Library (libitm).
+
+   Libitm is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Provide access to the futex system call.  */
+
+#ifndef GTM_FUTEX_H
+#define GTM_FUTEX_H 1
+
+namespace GTM HIDDEN {
+
+#include "futex_bits.h"
+
+extern void futex_wait (int *addr, int val);
+extern long futex_wake (int *addr, int count);
+
+}
+
+#endif /* GTM_FUTEX_H */
diff --git a/libitm/config/linux/rwlock.cc b/libitm/config/linux/rwlock.cc
new file mode 100644
index 0000000..c1e935e
--- /dev/null
+++ b/libitm/config/linux/rwlock.cc
@@ -0,0 +1,235 @@
+/* Copyright (C) 2011 Free Software Foundation, Inc.
+   Contributed by Torvald Riegel <triegel@redhat.com>.
+
+   This file is part of the GNU Transactional Memory Library (libitm).
+
+   Libitm is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "libitm_i.h"
+#include "futex.h"
+#include <limits.h>
+
+namespace GTM HIDDEN {
+
+// Acquire a RW lock for reading.
+
+void
+gtm_rwlock::read_lock (gtm_thread *tx)
+{
+  for (;;)
+    {
+      // Fast path: first announce our intent to read, then check for
+      // conflicting intents to write. The barrier makes sure that this
+      // happens in exactly this order.
+      tx->shared_state = 0;
+      __sync_synchronize();
+      if (likely(writers == 0))
+	return;
+
+      // There seems to be an active, waiting, or confirmed writer, so enter
+      // the futex-based slow path.
+
+      // Before waiting, we clear our read intent check whether there are any
+      // writers that might potentially wait for readers. If so, wake them.
+      // We need the barrier here for the same reason that we need it in
+      // read_unlock().
+      // TODO Potentially too many wake-ups. See comments in read_unlock().
+      tx->shared_state = ~(typeof tx->shared_state)0;
+      __sync_synchronize();
+      if (writer_readers > 0)
+	{
+	  writer_readers = 0;
+	  futex_wake(&writer_readers, 1);
+	}
+
+      // Signal that there are waiting readers and wait until there is no
+      // writer anymore.
+      // TODO Spin here on writers for a while. Consider whether we woke
+      // any writers before?
+      while (writers)
+	{
+	  // An active writer. Wait until it has finished. To avoid lost
+	  // wake-ups, we need to use Dekker-like synchronization.
+	  // Note that we cannot reset readers to zero when we see that there
+	  // are no writers anymore after the barrier because this pending
+	  // store could then lead to lost wake-ups at other readers.
+	  readers = 1;
+	  __sync_synchronize();
+	  if (writers)
+	    futex_wait(&readers, 1);
+	}
+
+      // And we try again to acquire a read lock.
+    }
+}
+
+
+// Acquire a RW lock for writing. Generic version that also works for
+// upgrades.
+// Note that an upgrade might fail (and thus waste previous work done during
+// this transaction) if there is another thread that tried to go into serial
+// mode earlier (i.e., upgrades do not have higher priority than pure writers).
+// However, this seems rare enough to not consider it further as we need both
+// a non-upgrade writer and a writer to happen to switch to serial mode
+// concurrently. If we'd want to handle this, a writer waiting for readers
+// would have to coordinate with later arriving upgrades and hand over the
+// lock to them, including the the reader-waiting state. We can try to support
+// this if this will actually happen often enough in real workloads.
+
+bool
+gtm_rwlock::write_lock_generic (gtm_thread *tx)
+{
+  // Try to acquire the write lock.
+  unsigned int w;
+  if (unlikely((w = __sync_val_compare_and_swap(&writers, 0, 1)) != 0))
+    {
+      // If this is an upgrade, we must not wait for other writers or
+      // upgrades.
+      if (tx != 0)
+	return false;
+
+      // There is already a writer. If there are no other waiting writers,
+      // switch to contended mode.
+      // Note that this is actually an atomic exchange, not a TAS. Also,
+      // it's only guaranteed to have acquire semantics, whereas we need a
+      // full barrier to make the Dekker-style synchronization work. However,
+      // we rely on the xchg being a full barrier on the architectures that we
+      // consider here.
+      // ??? Use C++0x atomics as soon as they are available.
+      if (w != 2)
+	w = __sync_lock_test_and_set(&writers, 2);
+      while (w != 0)
+	{
+	  futex_wait(&writers, 2);
+	  w = __sync_lock_test_and_set(&writers, 2);
+	}
+    }
+
+  // We have acquired the writer side of the R/W lock. Now wait for any
+  // readers that might still be active.
+  // We don't need an extra barrier here because the CAS and the xchg
+  // operations have full barrier semantics already.
+
+  // If this is an upgrade, we are not a reader anymore. This is only safe to
+  // do after we have acquired the writer lock.
+  // TODO In the worst case, this requires one wait/wake pair for each
+  // active reader. Reduce this!
+  if (tx != 0)
+    tx->shared_state = ~(typeof tx->shared_state)0;
+
+  for (gtm_thread *it = gtm_thread::list_of_threads; it != 0;
+      it = it->next_thread)
+    {
+      // Use a loop here to check reader flags again after waiting.
+      while (it->shared_state != ~(typeof it->shared_state)0)
+	{
+	  // An active reader. Wait until it has finished. To avoid lost
+	  // wake-ups, we need to use Dekker-like synchronization.
+	  // Note that we can reset writer_readers to zero when we see after
+	  // the barrier that the reader has finished in the meantime;
+	  // however, this is only possible because we are the only writer.
+	  // TODO Spin for a while on this reader flag.
+	  writer_readers = 1;
+	  __sync_synchronize();
+	  if (it->shared_state != ~(typeof it->shared_state)0)
+	    futex_wait(&writer_readers, 1);
+	  else
+	    writer_readers = 0;
+	}
+    }
+
+  return true;
+}
+
+// Acquire a RW lock for writing.
+
+void
+gtm_rwlock::write_lock ()
+{
+  write_lock_generic (0);
+}
+
+
+// Upgrade a RW lock that has been locked for reading to a writing lock.
+// Do this without possibility of another writer incoming.  Return false
+// if this attempt fails (i.e. another thread also upgraded).
+
+bool
+gtm_rwlock::write_upgrade (gtm_thread *tx)
+{
+  return write_lock_generic (tx);
+}
+
+
+// Release a RW lock from reading.
+
+void
+gtm_rwlock::read_unlock (gtm_thread *tx)
+{
+  tx->shared_state = ~(typeof tx->shared_state)0;
+
+  // If there is a writer waiting for readers, wake it up. We need the barrier
+  // to avoid lost wake-ups.
+  // ??? We might not be the last active reader, so the wake-up might happen
+  // too early. How do we avoid this without slowing down readers too much?
+  // Each reader could scan the list of txns for other active readers but
+  // this can result in many cache misses. Use combining instead?
+  // TODO Sends out one wake-up for each reader in the worst case.
+  __sync_synchronize();
+  if (unlikely(writer_readers > 0))
+    {
+      writer_readers = 0;
+      futex_wake(&writer_readers, 1);
+    }
+}
+
+
+// Release a RW lock from writing.
+
+void
+gtm_rwlock::write_unlock ()
+{
+  // This is supposed to be a full barrier.
+  if (__sync_fetch_and_sub(&writers, 1) == 2)
+    {
+      // There might be waiting writers, so wake them.
+      writers = 0;
+      if (futex_wake(&writers, 1) == 0)
+	{
+	  // If we did not wake any waiting writers, we might indeed be the
+	  // last writer (this can happen because write_lock_generic()
+	  // exchanges 0 or 1 to 2 and thus might go to contended mode even if
+	  // no other thread holds the write lock currently). Therefore, we
+	  // have to wake up readers here as well.
+	  futex_wake(&readers, INT_MAX);
+	}
+      return;
+    }
+  // No waiting writers, so wake up all waiting readers.
+  // Because the fetch_and_sub is a full barrier already, we don't need
+  // another barrier here (as in read_unlock()).
+  if (readers > 0)
+    {
+      readers = 0;
+      futex_wake(&readers, INT_MAX);
+    }
+}
+
+} // namespace GTM
diff --git a/libitm/config/linux/rwlock.h b/libitm/config/linux/rwlock.h
new file mode 100644
index 0000000..7e6229b
--- /dev/null
+++ b/libitm/config/linux/rwlock.h
@@ -0,0 +1,66 @@
+/* Copyright (C) 2011 Free Software Foundation, Inc.
+   Contributed by Torvald Riegel <triegel@redhat.com>.
+
+   This file is part of the GNU Transactional Memory Library (libitm).
+
+   Libitm is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef GTM_RWLOCK_H
+#define GTM_RWLOCK_H
+
+#include "common.h"
+
+namespace GTM HIDDEN {
+
+struct gtm_thread;
+
+// This datastructure is the blocking, futex-based version of the Dekker-style
+// reader-writer lock used to provide mutual exclusion between active and
+// serial transactions.
+// See libitm's documentation for further details.
+//
+// In this implementation, writers are given highest priority access but
+// read-to-write upgrades do not have a higher priority than writers.
+
+class gtm_rwlock
+{
+  // TODO Put futexes on different cachelines?
+  int writers;          // Writers' futex.
+  int writer_readers;   // A confirmed writer waits here for readers.
+  int readers;          // Readers wait here for writers (iff true).
+
+ public:
+  gtm_rwlock() : writers(0), writer_readers(0), readers(0) {};
+
+  void read_lock (gtm_thread *tx);
+  void read_unlock (gtm_thread *tx);
+
+  void write_lock ();
+  void write_unlock ();
+
+  bool write_upgrade (gtm_thread *tx);
+
+ protected:
+  bool write_lock_generic (gtm_thread *tx);
+};
+
+} // namespace GTM
+
+#endif // GTM_RWLOCK_H
diff --git a/libitm/config/linux/x86/futex_bits.h b/libitm/config/linux/x86/futex_bits.h
new file mode 100644
index 0000000..9a6b102
--- /dev/null
+++ b/libitm/config/linux/x86/futex_bits.h
@@ -0,0 +1,82 @@
+/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
+   Contributed by Richard Henderson <rth@redhat.com>.
+
+   This file is part of the GNU Transactional Memory Library (libitm).
+
+   Libitm is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifdef __x86_64__
+# ifndef SYS_futex
+#  define SYS_futex	202
+# endif
+
+static inline long
+sys_futex0 (int *addr, long op, long val)
+{
+  register long r10 __asm__("%r10") = 0;
+  long res;
+
+  __asm volatile ("syscall"
+		  : "=a" (res)
+		  : "0" (SYS_futex), "D" (addr), "S" (op), "d" (val), "r" (r10)
+		  : "r11", "rcx", "memory");
+
+  return res;
+}
+
+#else
+# ifndef SYS_futex
+#  define SYS_futex	240
+# endif
+
+# ifdef __PIC__
+
+static inline long
+sys_futex0 (int *addr, int op, int val)
+{
+  long res;
+
+  __asm volatile ("xchgl\t%%ebx, %2\n\t"
+		  "int\t$0x80\n\t"
+		  "xchgl\t%%ebx, %2"
+		  : "=a" (res)
+		  : "0"(SYS_futex), "r" (addr), "c"(op),
+		    "d"(val), "S"(0)
+		  : "memory");
+  return res;
+}
+
+# else
+
+static inline long
+sys_futex0 (int *addr, int op, int val)
+{
+  long res;
+
+  __asm volatile ("int $0x80"
+		  : "=a" (res)
+		  : "0"(SYS_futex), "b" (addr), "c"(op),
+		    "d"(val), "S"(0)
+		  : "memory");
+  return res;
+}
+
+# endif /* __PIC__ */
+#endif /* __x86_64__ */
diff --git a/libitm/config/linux/x86/tls.h b/libitm/config/linux/x86/tls.h
new file mode 100644
index 0000000..01f7c27
--- /dev/null
+++ b/libitm/config/linux/x86/tls.h
@@ -0,0 +1,105 @@
+/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
+   Contributed by Richard Henderson <rth@redhat.com>.
+
+   This file is part of the GNU Transactional Memory Library (libitm).
+
+   Libitm is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef LIBITM_X86_TLS_H
+#define LIBITM_X86_TLS_H 1
+
+#if defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 10)
+/* Use slots in the TCB head rather than __thread lookups.
+   GLIBC has reserved words 10 through 13 for TM.  */
+#define HAVE_ARCH_GTM_THREAD 1
+#define HAVE_ARCH_GTM_THREAD_DISP 1
+#endif
+
+#include "config/generic/tls.h"
+
+#if defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 10)
+namespace GTM HIDDEN {
+
+#ifdef __x86_64__
+#ifdef __LP64__
+# define SEG_READ(OFS)		"movq\t%%fs:(" #OFS "*8),%0"
+# define SEG_WRITE(OFS)		"movq\t%0,%%fs:(" #OFS "*8)"
+# define SEG_DECODE_READ(OFS)	SEG_READ(OFS) "\n\t" \
+				"rorq\t$17,%0\n\t" \
+				"xorq\t%%fs:48,%0"
+# define SEG_ENCODE_WRITE(OFS)	"xorq\t%%fs:48,%0\n\t" \
+				"rolq\t$17,%0\n\t" \
+				SEG_WRITE(OFS)
+#else
+// For X32.
+# define SEG_READ(OFS)          "movl\t%%fs:(" #OFS "*4),%0"
+# define SEG_WRITE(OFS)         "movl\t%0,%%fs:(" #OFS "*4)"
+# define SEG_DECODE_READ(OFS)   SEG_READ(OFS) "\n\t" \
+				"rorl\t$9,%0\n\t" \
+				"xorl\t%%fs:24,%0"
+# define SEG_ENCODE_WRITE(OFS)  "xorl\t%%fs:24,%0\n\t" \
+				"roll\t$9,%0\n\t" \
+				SEG_WRITE(OFS)
+#endif
+#else
+# define SEG_READ(OFS)  "movl\t%%gs:(" #OFS "*4),%0"
+# define SEG_WRITE(OFS) "movl\t%0,%%gs:(" #OFS "*4)"
+# define SEG_DECODE_READ(OFS)	SEG_READ(OFS) "\n\t" \
+				"rorl\t$9,%0\n\t" \
+				"xorl\t%%gs:24,%0"
+# define SEG_ENCODE_WRITE(OFS)	"xorl\t%%gs:24,%0\n\t" \
+				"roll\t$9,%0\n\t" \
+				SEG_WRITE(OFS)
+#endif
+
+static inline struct gtm_thread *gtm_thr(void)
+{
+  struct gtm_thread *r;
+  asm volatile (SEG_READ(10) : "=r"(r));
+  return r;
+}
+
+static inline void set_gtm_thr(struct gtm_thread *x)
+{
+  asm volatile (SEG_WRITE(10) : : "r"(x));
+}
+
+static inline struct abi_dispatch *abi_disp(void)
+{
+  struct abi_dispatch *r;
+  asm volatile (SEG_DECODE_READ(11) : "=r"(r));
+  return r;
+}
+
+static inline void set_abi_disp(struct abi_dispatch *x)
+{
+  void *scratch;
+  asm volatile (SEG_ENCODE_WRITE(11) : "=r"(scratch) : "0"(x));
+}
+
+#undef SEG_READ
+#undef SEG_WRITE
+#undef SEG_DECODE_READ
+#undef SEG_ENCODE_WRITE
+
+} // namespace GTM
+#endif /* >= GLIBC 2.10 */
+
+#endif // LIBITM_X86_TLS_H