aboutsummaryrefslogtreecommitdiff
path: root/nptl
diff options
context:
space:
mode:
authorAndi Kleen <ak@linux.intel.com>2014-03-17 05:48:28 -0700
committerAndi Kleen <ak@linux.intel.com>2014-06-13 12:49:03 -0700
commita832bdd36203fcb37fa5ad25200ef3c1ae205efe (patch)
tree045064e8ee765c267caab8a76a0a014c39acb787 /nptl
parent995a46bbfba9964e328e3947130919d8bd3cd62a (diff)
downloadglibc-a832bdd36203fcb37fa5ad25200ef3c1ae205efe.zip
glibc-a832bdd36203fcb37fa5ad25200ef3c1ae205efe.tar.gz
glibc-a832bdd36203fcb37fa5ad25200ef3c1ae205efe.tar.bz2
Add a fast path for C rd/wrlock v2
One difference of the C versions to the assembler wr/rdlock is that the C compiler saves some registers which are unnecessary for the fast path in the prologue of the functions. Split the uncontended fast path out into a separate function. Only when contention is detected is the full featured function called. This makes the fast path code (nearly) identical to the assembler version, and gives uncontended performance within a few cycles. v2: Rename some functions and add space.
Diffstat (limited to 'nptl')
-rw-r--r--nptl/pthread_rwlock_rdlock.c88
-rw-r--r--nptl/pthread_rwlock_wrlock.c59
2 files changed, 103 insertions, 44 deletions
diff --git a/nptl/pthread_rwlock_rdlock.c b/nptl/pthread_rwlock_rdlock.c
index 3773f7d..1df0327 100644
--- a/nptl/pthread_rwlock_rdlock.c
+++ b/nptl/pthread_rwlock_rdlock.c
@@ -24,39 +24,16 @@
#include <stap-probe.h>
-/* Acquire read lock for RWLOCK. */
-int
-__pthread_rwlock_rdlock (rwlock)
- pthread_rwlock_t *rwlock;
+/* Acquire read lock for RWLOCK. Slow path. */
+static int __attribute__((noinline))
+__pthread_rwlock_rdlock_slow (pthread_rwlock_t *rwlock)
{
int result = 0;
- LIBC_PROBE (rdlock_entry, 1, rwlock);
-
- /* Make sure we are alone. */
- lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
+ /* Lock is taken in caller. */
while (1)
{
- /* Get the rwlock if there is no writer... */
- if (rwlock->__data.__writer == 0
- /* ...and if either no writer is waiting or we prefer readers. */
- && (!rwlock->__data.__nr_writers_queued
- || PTHREAD_RWLOCK_PREFER_READER_P (rwlock)))
- {
- /* Increment the reader counter. Avoid overflow. */
- if (__glibc_unlikely (++rwlock->__data.__nr_readers == 0))
- {
- /* Overflow on number of readers. */
- --rwlock->__data.__nr_readers;
- result = EAGAIN;
- }
- else
- LIBC_PROBE (rdlock_acquire_read, 1, rwlock);
-
- break;
- }
-
/* Make sure we are not holding the rwlock as a writer. This is
a deadlock situation we recognize and report. */
if (__builtin_expect (rwlock->__data.__writer
@@ -88,6 +65,25 @@ __pthread_rwlock_rdlock (rwlock)
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
--rwlock->__data.__nr_readers_queued;
+
+ /* Get the rwlock if there is no writer... */
+ if (rwlock->__data.__writer == 0
+ /* ...and if either no writer is waiting or we prefer readers. */
+ && (!rwlock->__data.__nr_writers_queued
+ || PTHREAD_RWLOCK_PREFER_READER_P (rwlock)))
+ {
+ /* Increment the reader counter. Avoid overflow. */
+ if (__glibc_unlikely (++rwlock->__data.__nr_readers == 0))
+ {
+ /* Overflow on number of readers. */
+ --rwlock->__data.__nr_readers;
+ result = EAGAIN;
+ }
+ else
+ LIBC_PROBE (rdlock_acquire_read, 1, rwlock);
+
+ break;
+ }
}
/* We are done, free the lock. */
@@ -96,5 +92,43 @@ __pthread_rwlock_rdlock (rwlock)
return result;
}
+
+/* Fast path of acquiring read lock on RWLOCK. */
+
+int
+__pthread_rwlock_rdlock (pthread_rwlock_t *rwlock)
+{
+ int result = 0;
+
+ LIBC_PROBE (rdlock_entry, 1, rwlock);
+
+ /* Make sure we are alone. */
+ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
+
+ /* Get the rwlock if there is no writer... */
+ if (rwlock->__data.__writer == 0
+ /* ...and if either no writer is waiting or we prefer readers. */
+ && (!rwlock->__data.__nr_writers_queued
+ || PTHREAD_RWLOCK_PREFER_READER_P (rwlock)))
+ {
+ /* Increment the reader counter. Avoid overflow. */
+ if (__glibc_unlikely (++rwlock->__data.__nr_readers == 0))
+ {
+ /* Overflow on number of readers. */
+ --rwlock->__data.__nr_readers;
+ result = EAGAIN;
+ }
+ else
+ LIBC_PROBE (rdlock_acquire_read, 1, rwlock);
+
+ /* We are done, free the lock. */
+ lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
+
+ return result;
+ }
+
+ return __pthread_rwlock_rdlock_slow (rwlock);
+}
+
weak_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
hidden_def (__pthread_rwlock_rdlock)
diff --git a/nptl/pthread_rwlock_wrlock.c b/nptl/pthread_rwlock_wrlock.c
index 1613d45..de54e51 100644
--- a/nptl/pthread_rwlock_wrlock.c
+++ b/nptl/pthread_rwlock_wrlock.c
@@ -25,29 +25,15 @@
/* Acquire write lock for RWLOCK. */
-int
-__pthread_rwlock_wrlock (rwlock)
- pthread_rwlock_t *rwlock;
+static int __attribute__((noinline))
+__pthread_rwlock_wrlock_slow (pthread_rwlock_t *rwlock)
{
int result = 0;
- LIBC_PROBE (wrlock_entry, 1, rwlock);
-
- /* Make sure we are alone. */
- lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
+ /* Caller has taken the lock. */
while (1)
{
- /* Get the rwlock if there is no writer and no reader. */
- if (rwlock->__data.__writer == 0 && rwlock->__data.__nr_readers == 0)
- {
- /* Mark self as writer. */
- rwlock->__data.__writer = THREAD_GETMEM (THREAD_SELF, tid);
-
- LIBC_PROBE (wrlock_acquire_write, 1, rwlock);
- break;
- }
-
/* Make sure we are not holding the rwlock as a writer. This is
a deadlock situation we recognize and report. */
if (__builtin_expect (rwlock->__data.__writer
@@ -80,6 +66,16 @@ __pthread_rwlock_wrlock (rwlock)
/* To start over again, remove the thread from the writer list. */
--rwlock->__data.__nr_writers_queued;
+
+ /* Get the rwlock if there is no writer and no reader. */
+ if (rwlock->__data.__writer == 0 && rwlock->__data.__nr_readers == 0)
+ {
+ /* Mark self as writer. */
+ rwlock->__data.__writer = THREAD_GETMEM (THREAD_SELF, tid);
+
+ LIBC_PROBE (wrlock_acquire_write, 1, rwlock);
+ break;
+ }
}
/* We are done, free the lock. */
@@ -88,5 +84,34 @@ __pthread_rwlock_wrlock (rwlock)
return result;
}
+/* Fast path of acquiring write lock for RWLOCK. */
+
+int
+__pthread_rwlock_wrlock (pthread_rwlock_t *rwlock)
+{
+ LIBC_PROBE (wrlock_entry, 1, rwlock);
+
+ /* Make sure we are alone. */
+ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
+
+ /* Get the rwlock if there is no writer and no reader. */
+ if (__glibc_likely((rwlock->__data.__writer |
+ rwlock->__data.__nr_readers) == 0))
+ {
+ /* Mark self as writer. */
+ rwlock->__data.__writer = THREAD_GETMEM (THREAD_SELF, tid);
+
+ LIBC_PROBE (wrlock_acquire_write, 1, rwlock);
+
+ /* We are done, free the lock. */
+ lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
+
+ return 0;
+ }
+
+ return __pthread_rwlock_wrlock_slow (rwlock);
+}
+
+
weak_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
hidden_def (__pthread_rwlock_wrlock)