Add adaptive elision to rwlocks

This patch relies on the C version of the rwlocks posted earlier.
With C rwlocks it is very straight forward to do adaptive elision
using TSX. It is based on the infrastructure added earlier
for mutexes, but uses its own elision macros. The macros
are fairly general purpose and could be used for other
elision purposes too.

This version is much cleaner than the earlier assembler based
version, and in particular implements adaptation which makes
it safer.

I changed the behavior slightly to not require any changes
in the test suite and fully conform to all expected
behaviors (generally at the cost of not eliding in
various situations). In particular this means the timedlock
variants are not elided.  Nested trylock aborts.

6 files changed, 36 insertions, 0 deletions

diff --git a/nptl/pthread_rwlock_rdlock.c b/nptl/pthread_rwlock_rdlock.c
index 1df0327..6eb9e09 100644
--- a/nptl/pthread_rwlock_rdlock.c
+++ b/nptl/pthread_rwlock_rdlock.c
@@ -22,6 +22,7 @@
 #include <pthread.h>
 #include <pthreadP.h>
 #include <stap-probe.h>
+#include <elide.h>
 
 
 /* Acquire read lock for RWLOCK.  Slow path.  */
@@ -102,6 +103,12 @@ __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock)
 
   LIBC_PROBE (rdlock_entry, 1, rwlock);
 
+  if (ELIDE_LOCK (rwlock->__data.__rwelision,
+		  rwlock->__data.__lock == 0
+		  && rwlock->__data.__writer == 0
+		  && rwlock->__data.__nr_readers == 0))
+    return 0;
+
   /* Make sure we are alone.  */
   lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
 
diff --git a/nptl/pthread_rwlock_tryrdlock.c b/nptl/pthread_rwlock_tryrdlock.c
index f7b1e6b..cf8d68e 100644
--- a/nptl/pthread_rwlock_tryrdlock.c
+++ b/nptl/pthread_rwlock_tryrdlock.c
@@ -19,6 +19,7 @@
 #include <errno.h>
 #include "pthreadP.h"
 #include <lowlevellock.h>
+#include <elide.h>
 
 
 int
@@ -26,6 +27,12 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock)
 {
   int result = EBUSY;
 
+  if (ELIDE_TRYLOCK (rwlock->__data.__rwelision,
+		     rwlock->__data.__lock == 0
+		     && rwlock->__data.__nr_readers == 0
+		     && rwlock->__data.__writer, 0))
+    return 0;
+
   lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
 
   if (rwlock->__data.__writer == 0
diff --git a/nptl/pthread_rwlock_trywrlock.c b/nptl/pthread_rwlock_trywrlock.c
index 106f157..0291fc9 100644
--- a/nptl/pthread_rwlock_trywrlock.c
+++ b/nptl/pthread_rwlock_trywrlock.c
@@ -19,6 +19,7 @@
 #include <errno.h>
 #include "pthreadP.h"
 #include <lowlevellock.h>
+#include <elide.h>
 
 
 int
@@ -26,6 +27,12 @@ __pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock)
 {
   int result = EBUSY;
 
+  if (ELIDE_TRYLOCK (rwlock->__data.__rwelision,
+		     rwlock->__data.__lock == 0
+		     && rwlock->__data.__nr_readers == 0
+		     && rwlock->__data.__writer, 1))
+    return 0;
+
   lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
 
   if (rwlock->__data.__writer == 0 && rwlock->__data.__nr_readers == 0)
diff --git a/nptl/pthread_rwlock_unlock.c b/nptl/pthread_rwlock_unlock.c
index d492383..3ebddeb 100644
--- a/nptl/pthread_rwlock_unlock.c
+++ b/nptl/pthread_rwlock_unlock.c
@@ -22,6 +22,8 @@
 #include <pthread.h>
 #include <pthreadP.h>
 #include <stap-probe.h>
+#include <elide.h>
+
 
 /* Unlock RWLOCK.  */
 int
@@ -29,6 +31,10 @@ __pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
 {
   LIBC_PROBE (rwlock_unlock, 1, rwlock);
 
+  if (ELIDE_UNLOCK (rwlock->__data.__writer == 0
+		    && rwlock->__data.__nr_readers == 0))
+    return 0;
+
   lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
   if (rwlock->__data.__writer)
     rwlock->__data.__writer = 0;
diff --git a/nptl/pthread_rwlock_wrlock.c b/nptl/pthread_rwlock_wrlock.c
index de54e51..91ad82a 100644
--- a/nptl/pthread_rwlock_wrlock.c
+++ b/nptl/pthread_rwlock_wrlock.c
@@ -22,6 +22,7 @@
 #include <pthread.h>
 #include <pthreadP.h>
 #include <stap-probe.h>
+#include <elide.h>
 
 
 /* Acquire write lock for RWLOCK.  */
@@ -91,6 +92,12 @@ __pthread_rwlock_wrlock (pthread_rwlock_t *rwlock)
 {
   LIBC_PROBE (wrlock_entry, 1, rwlock);
 
+  if (ELIDE_LOCK (rwlock->__data.__rwelision,
+		  rwlock->__data.__lock == 0
+		  && rwlock->__data.__writer == 0
+		  && rwlock->__data.__nr_readers == 0))
+    return 0;
+
   /* Make sure we are alone.  */
   lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
 
diff --git a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h
index d70f8b3..c9f1c83 100644
--- a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h
@@ -214,6 +214,8 @@ typedef union
   long int __align;
 } pthread_rwlock_t;
 
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
 typedef union
 {
   char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];