* sysdeps/unix/sysv/linux/x86_64/lowlevellock.S (__lll_timedlock_wait):

	Store 2 before returning ETIMEDOUT.
	* sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S: Likewise
	* sysdeps/unix/sysv/linux/lowlevellock.c: Likewise.
	(__lll_lock_wait_private): Optimize.
	(__lll_lock_wait): Likewise.

3 files changed, 56 insertions, 105 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
index 6d3943b..9c8a68f 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
@@ -184,6 +184,12 @@ __lll_timedlock_wait:
 	movl	%ecx, %ebp
 	movl	%edx, %edi
 
+	movl	$2, %edx
+	xchgl	%edx, (%ebp)
+
+	test	%edx, %edx
+	je	6f
+
 1:
 	/* Get current time.  */
 	movl	%esp, %ebx
@@ -203,36 +209,30 @@ __lll_timedlock_wait:
 	addl	$1000000000, %edx
 	subl	$1, %ecx
 4:	testl	%ecx, %ecx
-	js	9f		/* Time is already up.  */
+	js	2f		/* Time is already up.  */
 
 	/* Store relative timeout.  */
 	movl	%ecx, (%esp)
 	movl	%edx, 4(%esp)
 
+	/* Futex call.  */
 	movl	%ebp, %ebx
-
-	movl	$1, %eax
 	movl	$2, %edx
-	LOCK
-	cmpxchgl %edx, (%ebx)
-
-	testl	%eax, %eax
-	je	8f
-
-	/* Futex call.  */
 	movl	%esp, %esi
 	movl	16(%esp), %ecx
 	LOAD_FUTEX_WAIT (%ecx)
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
-	movl	%eax, %ecx
 
-8:				/* NB: %edx == 2 */
-	xorl	%eax, %eax
-	LOCK
-	cmpxchgl %edx, (%ebx)
+	/* NB: %edx == 2 */
+	xchgl	%edx, (%ebp)
 
-	jnz	7f
+	testl	%edx, %edx
+	je	6f
+
+	cmpl	$-ETIMEDOUT, %eax
+	jne	1b
+2:	movl	$ETIMEDOUT, %edx
 
 6:	addl	$8, %esp
 	cfi_adjust_cfa_offset(-8)
@@ -248,33 +248,11 @@ __lll_timedlock_wait:
 	popl	%edi
 	cfi_adjust_cfa_offset(-4)
 	cfi_restore(%edi)
+	movl	%edx, %eax
 	ret
 
 3:	movl	$EINVAL, %eax
 	ret
-
-	cfi_adjust_cfa_offset(24)
-	cfi_offset(%edi, -8)
-	cfi_offset(%esi, -12)
-	cfi_offset(%ebx, -16)
-	cfi_offset(%ebp, -20)
-	/* Check whether the time expired.  */
-7:	cmpl	$-ETIMEDOUT, %ecx
-	je	5f
-
-	/* Make sure the current holder knows we are going to sleep.  */
-	movl	%edx, %eax
-	xchgl	%eax, (%ebx)
-	testl	%eax, %eax
-	jz	6b
-	jmp	1b
-
-5:	movl	$ETIMEDOUT, %eax
-	jmp	6b
-
-9:	movl	$-ETIMEDOUT, %ecx
-	movl	$2, %edx
-	jmp	8b
 	cfi_endproc
 	.size	__lll_timedlock_wait,.-__lll_timedlock_wait
 #endif
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevellock.c b/nptl/sysdeps/unix/sysv/linux/lowlevellock.c
index c2cdf3a..01c4f48 100644
--- a/nptl/sysdeps/unix/sysv/linux/lowlevellock.c
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevellock.c
@@ -27,13 +27,11 @@
 void
 __lll_lock_wait_private (int *futex)
 {
-  do
-    {
-      int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1);
-      if (oldval != 0)
-	lll_futex_wait (futex, 2, LLL_PRIVATE);
-    }
-  while (atomic_compare_and_exchange_bool_acq (futex, 2, 0) != 0);
+  if (*futex == 2)
+    lll_futex_wait (futex, 2, LLL_PRIVATE);
+
+  while (atomic_exchange_acq (futex, 2) != 0)
+    lll_futex_wait (futex, 2, LLL_PRIVATE);
 }
 
 
@@ -42,13 +40,11 @@ __lll_lock_wait_private (int *futex)
 void
 __lll_lock_wait (int *futex, int private)
 {
-  do
-    {
-      int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1);
-      if (oldval != 0)
-	lll_futex_wait (futex, 2, private);
-    }
-  while (atomic_compare_and_exchange_bool_acq (futex, 2, 0) != 0);
+  if (*futex == 2)
+    lll_futex_wait (futex, 2, private);
+
+  while (atomic_exchange_acq (futex, 2) != 0)
+    lll_futex_wait (futex, 2, private);
 }
 
 
@@ -59,8 +55,8 @@ __lll_timedlock_wait (int *futex, const struct timespec *abstime, int private)
   if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
     return EINVAL;
 
-  struct timespec rt;
-  do
+  /* Try locking.  */
+  while (atomic_exchange_acq (futex, 2) != 0)
     {
       struct timeval tv;
 
@@ -68,6 +64,7 @@ __lll_timedlock_wait (int *futex, const struct timespec *abstime, int private)
       (void) __gettimeofday (&tv, NULL);
 
       /* Compute relative timeout.  */
+      struct timespec rt;
       rt.tv_sec = abstime->tv_sec - tv.tv_sec;
       rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
       if (rt.tv_nsec < 0)
@@ -76,21 +73,14 @@ __lll_timedlock_wait (int *futex, const struct timespec *abstime, int private)
 	  --rt.tv_sec;
 	}
 
-      /* If timed out do not go to sleep.  */
-      if (__builtin_expect (rt.tv_sec >= 0, 1))
-	{
-	  /* Wait.  */
-	  int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1);
-	  if (oldval != 0)
-	    lll_futex_timed_wait (futex, 2, &rt, private);
-	}
+      if (rt.tv_sec < 0)
+	return ETIMEDOUT;
 
-      if (atomic_compare_and_exchange_bool_acq (futex, 2, 0) == 0)
-	return 0;
+      /* Wait.  */
+      lll_futex_timed_wait (futex, 2, &rt, private);
     }
-  while (rt.tv_sec >= 0);
 
-  return ETIMEDOUT;
+  return 0;
 }
 
 
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
index a5ad88c..4505e2c 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
@@ -178,6 +178,12 @@ __lll_timedlock_wait:
 	movq	%rdi, %r12
 	movq	%rdx, %r13
 
+	movl	$2, %edx
+	xchgl	%edx, (%r12)
+
+	testl	%edx, %edx
+	je	6f
+
 1:
 	/* Get current time.  */
 	movq	%rsp, %rdi
@@ -199,35 +205,31 @@ __lll_timedlock_wait:
 	addq	$1000000000, %rsi
 	decq	%rdi
 4:	testq	%rdi, %rdi
-	movq	$-ETIMEDOUT, %rcx
-	movl	$2, %edx
-	js	8f		/* Time is already up.  */
+	js	2f		/* Time is already up.  */
 
-	/* Futex call.  */
-	movq	%rdi, (%rsp)	/* Store relative timeout.  */
+	/* Store relative timeout.  */
+	movq	%rdi, (%rsp)
 	movq	%rsi, 8(%rsp)
 
+	/* Futex call.  */
+	movl	$2, %edx
 	movl	$1, %eax
-				/* NB: $edx has been loaded early.  */
-	LOCK
-	cmpxchgl %edx, (%r12)
-
-	testl	%eax, %eax
-	je	8f
-
 	movq	%rsp, %r10
 	movl	24(%rsp), %esi
 	LOAD_FUTEX_WAIT (%esi)
 	movq	%r12, %rdi
 	movl	$SYS_futex, %eax
 	syscall
-	movq	%rax, %rcx
 
-8:				/* NB: %edx == 2 */
-	xorl	%eax, %eax
-	LOCK
-	cmpxchgl %edx, (%r12)
-	jnz	7f
+	/* NB: %edx == 2 */
+	xchgl	%edx, (%r12)
+
+	testl	%edx, %edx
+	je	6f
+
+	cmpl	$-ETIMEDOUT, %eax
+	jne	1b
+2:	movl	$ETIMEDOUT, %edx
 
 6:	addq	$32, %rsp
 	cfi_adjust_cfa_offset(-32)
@@ -246,30 +248,11 @@ __lll_timedlock_wait:
 	popq	%r8
 	cfi_adjust_cfa_offset(-8)
 	cfi_restore(%r8)
+	movl	%edx, %eax
 	retq
 
 3:	movl	$EINVAL, %eax
 	retq
-
-	cfi_adjust_cfa_offset(72)
-	cfi_offset(%r8, -16)
-	cfi_offset(%r9, -24)
-	cfi_offset(%r12, -32)
-	cfi_offset(%r13, -40)
-	cfi_offset(%r14, -48)
-	/* Check whether the time expired.  */
-7:	cmpq	$-ETIMEDOUT, %rcx
-	je	5f
-
-	/* Make sure the current holder knows we are going to sleep.  */
-	movl	%edx, %eax
-	xchgl	%eax, (%rdi)
-	testl	%eax, %eax
-	jz	6b
-	jmp	1b
-
-5:	movl	$ETIMEDOUT, %eax
-	jmp	6b
 	cfi_endproc
 	.size	__lll_timedlock_wait,.-__lll_timedlock_wait
 #endif