Add sigstack handling to Linux ____longjmp_chk on powerpc.

6 files changed, 547 insertions, 11 deletions

diff --git a/nptl/pthread_rwlock_init.c b/nptl/pthread_rwlock_init.c
index 27f25ac..95d9da4 100644
--- a/nptl/pthread_rwlock_init.c
+++ b/nptl/pthread_rwlock_init.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2007, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -37,13 +37,7 @@ __pthread_rwlock_init (rwlock, attr)
 
   iattr = ((const struct pthread_rwlockattr *) attr) ?: &default_attr;
 
-  rwlock->__data.__lock = 0;
-  rwlock->__data.__nr_readers = 0;
-  rwlock->__data.__readers_wakeup = 0;
-  rwlock->__data.__writer_wakeup = 0;
-  rwlock->__data.__nr_readers_queued = 0;
-  rwlock->__data.__nr_writers_queued = 0;
-  rwlock->__data.__writer = 0;
+  memset (rwlock, '\0', sizeof (*rwlock));
 
   rwlock->__data.__flags
     = iattr->lockkind == PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP;
@@ -74,9 +68,6 @@ __pthread_rwlock_init (rwlock, attr)
 					      header.private_futex));
 #endif
 
-  rwlock->__data.__pad1 = 0;
-  rwlock->__data.__pad2 = 0;
-
   return 0;
 }
 strong_alias (__pthread_rwlock_init, pthread_rwlock_init)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
index 35eb09c..54a47e1 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
@@ -31,6 +31,127 @@
 	.align	16
 __pthread_rwlock_rdlock:
 	cfi_startproc
+	movq	$NRW_RC, %rax
+	LOCK
+	xaddq	%rax, NRW_WORD(%rdi)
+	js	.Lundo
+
+.Lagain1:
+	movq	$(NRW_WL|NRW_WP), %rcx
+	movq	$(NRW_WL|NRW_WW_MASK), %rdx
+	testq	%rax, %rcx
+	setnz	%cl
+	testq	%rax, %rdx
+	setnz	%dl
+	orb	%cl, %dl
+	jnz	.Lwait
+
+	xorl	%eax, %eax
+	ret
+
+.Lwait:	movq	$NRW_WL, %rcx
+	testq	%rcx, %rax
+	jz	4f
+
+	movl	%fs:TID, %ecx
+	cmpl	%ecx, WRITER(%rdi)
+	je	.Ldeadlk
+
+4:	xorq	%r8, %r8
+	movq	$NRW_RC, %rdx
+	addq	%rdx, %rax
+	movq	$(NRW_WL|NRW_AR), %rcx
+	movq	$(NRW_WP|NRW_WW_MASK), %rdx
+	testq	%rax, %rcx
+	setz	%cl
+	testq	%rax, %rdx
+	setnz	%dl
+	testb	%dl, %cl
+	jz	1f
+
+	movq	$NRW_RW-NRW_RC, %rdx
+	addq	%rax, %rdx
+	LOCK
+	cmpxchgq %rdx, NRW_WORD(%rdi)
+	jnz	.Lagain1
+
+	movq	$NRW_RC_MASK, %rax
+	testq	%rdx, %rax
+	movl	$1, %r8d
+	jz	.Lwake_waiter
+
+1:	xorq	%r10, %r10
+	movl	$NRW_R_WAKEUP, %r9d
+	movl	$(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi
+	xorl	PSHARED(%rdi), %esi
+	leaq	NRW_WORD+4(%rdi), %rdi
+2:	shldq	$32, %rax, %rdx
+	movl	$__NR_futex, %eax
+	syscall
+	movq	-4(%rdi), %rax
+
+5:	movq	$(NRW_WL|NRW_WP), %rcx
+	movq	$(NRW_WL|NRW_WW_MASK), %rdx
+	testq	%rax, %rcx
+	setnz	%cl
+	testq	%rax, %rdx
+	setnz	%dl
+	orb	%cl, %dl
+	jnz	2b
+
+	testl	%r8d, %r8d
+	jz	3f
+
+	movq	$NRW_RC-NRW_RW, %rcx
+	addq	%rcx, %rdx
+	LOCK
+	cmpxchgq %rdx, -4(%rdi)
+	jnz	5b
+
+3:	xorl	%eax, %eax
+	ret
+
+.Lwake_waiter:
+	movq	%rdx, %r10
+	movl	$__NR_futex, %eax
+	movl	$(FUTEX_WAKE_BITSET|FUTEX_PRIVATE_FLAG), %esi
+	xorl	PSHARED(%rdi), %esi
+	leaq	NRW_WORD(%rdi), %rdi
+	movl	$1, %edx
+	movl	$NRW_W_WAKEUP, %r9d
+	syscall
+	leaq	-NRW_WORD(%rdi), %rdi
+	movq	%r10, %rdx
+	jmp	1b
+
+.Lundo:	movq	$-NRW_RC, %rcx
+	movq	%rax, %rdx
+	subq	%rcx, %rax
+	LOCK
+	cmpxchgq %rdx, NRW_WORD(%rdi)
+	jz	.Lret
+
+	movq	$NRW_RC_OVFL, %r8
+.Lagain2:
+	testq	%r8, %rax
+	jz	.Lagain1
+	leaq	(%rax,%rcx), %rdx
+	LOCK
+	cmpxchgq %rdx, NRW_WORD(%rdi)
+	jnz	.Lagain2
+
+.Lret:	movl	$EAGAIN, %eax
+	ret
+
+.Ldeadlk:
+	movq	$NRW_RC, %rdx
+	LOCK
+	subq	%rdx, NRW_WORD(%rdi)
+
+	movl	$EDEADLK, %eax
+	ret
+
+#if 0
 	xorq	%r10, %r10
 
 	/* Get the lock.  */
@@ -168,6 +289,7 @@ __pthread_rwlock_rdlock:
 	subq	$MUTEX, %rdi
 #endif
 	jmp	13b
+#endif
 	cfi_endproc
 	.size	__pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
 
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
index 23b218a..0fb925c 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
@@ -34,6 +34,143 @@
 	.align	16
 pthread_rwlock_timedrdlock:
 	cfi_startproc
+	movq	%rsi, %r10
+	movq	$NRW_RC, %rax
+	LOCK
+	xaddq	%rax, NRW_WORD(%rdi)
+	js	.Lundo
+
+.Lagain1:
+	movq	$(NRW_WL|NRW_WP), %rcx
+	movq	$(NRW_WL|NRW_WW_MASK), %rdx
+	testq	%rax, %rcx
+	setnz	%cl
+	testq	%rax, %rdx
+	setnz	%dl
+	orb	%cl, %dl
+	jnz	.Lwait
+
+	xorl	%eax, %eax
+	ret
+
+.Lwait:	movq	$NRW_WL, %rcx
+	testq	%rcx, %rax
+	jz	4f
+
+	movl	%fs:TID, %ecx
+	cmpl	%ecx, WRITER(%rdi)
+	je	.Ldeadlk
+
+4:	xorq	%r8, %r8
+	movq	$NRW_RC, %rdx
+	addq	%rdx, %rax
+	movq	$(NRW_WL|NRW_AR), %rcx
+	movq	$(NRW_WP|NRW_WW_MASK), %rdx
+	testq	%rax, %rcx
+	setz	%cl
+	testq	%rax, %rdx
+	setnz	%dl
+	testb	%dl, %cl
+	jz	1f
+
+	movq	$NRW_RW-NRW_RC, %rdx
+	addq	%rax, %rdx
+	LOCK
+	cmpxchgq %rdx, NRW_WORD(%rdi)
+	jnz	.Lagain1
+
+	movq	$NRW_RC_MASK, %rax
+	testq	%rdx, %rax
+	movq	$NRW_RW-NRW_RC, %r8
+	jz	.Lwake_waiter
+
+1:	movl	$NRW_R_WAKEUP, %r9d
+	leaq	NRW_WORD+4(%rdi), %rdi
+2:	movl	$(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG|FUTEX_CLOCK_REALTIME), %esi
+	xorl	PSHARED-(NRW_WORD+4)(%rdi), %esi
+	shldq	$32, %rax, %rdx
+	movl	$__NR_futex, %eax
+	syscall
+	movl	%eax, %esi
+	movq	-4(%rdi), %rax
+
+6:	movq	$(NRW_WL|NRW_WP), %rcx
+	movq	$(NRW_WL|NRW_WW_MASK), %rdx
+	testq	%rax, %rcx
+	setnz	%cl
+	testq	%rax, %rdx
+	setnz	%dl
+	orb	%cl, %dl
+	jz	5f
+
+	cmpl	$-EWOULDBLOCK, %esi
+	je	2b
+
+	movq	$-NRW_RC, %rdx
+	subq	%r8, %rdx
+	addq	%rax, %rdx
+	LOCK
+	cmpxchgq %rdx, -4(%rdi)
+	jnz	6b
+
+	negl	%esi
+	movl	%esi, %eax
+	ret
+
+5:	testl	%r8d, %r8d
+	jz	3f
+
+	movq	$NRW_RC-NRW_RW, %rcx
+	addq	%rcx, %rdx
+	LOCK
+	cmpxchgq %rdx, -4(%rdi)
+	jnz	6b
+
+3:	xorl	%eax, %eax
+	ret
+
+.Lwake_waiter:
+	movq	%rdx, %r10
+	movl	$__NR_futex, %eax
+	movl	$(FUTEX_WAKE_BITSET|FUTEX_PRIVATE_FLAG), %esi
+	xorl	PSHARED(%rdi), %esi
+	leaq	NRW_WORD(%rdi), %rdi
+	movl	$1, %edx
+	movl	$NRW_W_WAKEUP, %r9d
+	syscall
+	leaq	-NRW_WORD(%rdi), %rdi
+	movq	%r10, %rdx
+	jmp	1b
+
+.Lundo:	movq	$-NRW_RC, %rcx
+	movq	%rax, %rdx
+	subq	%rcx, %rax
+	LOCK
+	cmpxchgq %rdx, NRW_WORD(%rdi)
+	jz	.Lret
+
+	movq	$NRW_RC_OVFL, %r8
+.Lagain2:
+	testq	%r8, %rax
+	jz	.Lagain1
+	leaq	(%rax,%rcx), %rdx
+	LOCK
+	cmpxchgq %rdx, NRW_WORD(%rdi)
+	jnz	.Lagain2
+
+.Lret:	movl	$EAGAIN, %eax
+	ret
+
+.Ldeadlk:
+	movq	$NRW_RC, %rdx
+	LOCK
+	subq	%rdx, NRW_WORD(%rdi)
+
+	movl	$EDEADLK, %eax
+	ret
+
+
+#if 0
 	pushq	%r12
 	cfi_adjust_cfa_offset(8)
 	cfi_rel_offset(%r12, 0)
@@ -271,5 +408,6 @@ pthread_rwlock_timedrdlock:
 
 19:	movl	$EINVAL, %edx
 	jmp	9b
+#endif
 	cfi_endproc
 	.size	pthread_rwlock_timedrdlock,.-pthread_rwlock_timedrdlock
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
index cd867b6..14d7637 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
@@ -34,6 +34,109 @@
 	.align	16
 pthread_rwlock_timedwrlock:
 	cfi_startproc
+	movq	%rsi, %r10
+	movq	NRW_WORD(%rdi), %rax
+.Lagain:
+	movq	$(NRW_WW_MASK|NRW_WL|NRW_RC_MASK), %rdx
+	testq	%rdx, %rax
+	jnz	.Lwait
+
+	// XXX Probably make AR a don't-care for !WP.  Unconditionally set it
+	movq	$(NRW_AR|NRW_WL), %rdx
+	movq	$NRW_WL, %rcx
+	testq	$NRW_WP, %rax
+	cmovz	%rcx, %rdx
+	orq	%rax, %rdx
+	LOCK
+	cmpxchgq %rdx, NRW_WORD(%rdi)
+	jnz	.Lagain
+
+	movl	%fs:TID, %eax
+	movl	%eax, WRITER(%rdi)
+
+	xorl	%eax, %eax
+	ret
+
+.Lwait:	movq	$NRW_WL, %rcx
+	testq	%rcx, %rax
+	jz	1f
+
+	movl	%fs:TID, %eax
+	cmpl	%eax, WRITER(%rdi)
+	je	.Ldeadlk
+
+1:	leaq	NRW_WORD+4(%rdi), %rdi
+
+	movq	$NRW_WW, %rdx
+	movq	$NRW_WW_MASK, %rcx
+	addq	%rax, %rdx
+	testq	%rcx, %rdx
+	jz	.Lovfl
+
+	LOCK
+	cmpxchgq  %rdx, -4(%rdi)
+	jnz	.Lagain
+
+	movl	$NRW_W_WAKEUP, %r9d
+.Lwait2:
+	movl	$(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG|FUTEX_CLOCK_REALTIME), %esi
+	movl	$__NR_futex, %eax
+	xorl	PSHARED-(NRW_WORD+4)(%rdi), %esi
+	syscall
+	movl	%eax, %esi
+	movq	-4(%rdi), %rax
+
+.Lagain2:
+	movq	$(NRW_WL|NRW_RC_MASK), %rdx
+	movq	$(NRW_WL|NRW_AR), %rcx
+	testq	%rdx, %rax
+	movq	$NRW_AR, %rsi
+	setz	%cl
+	andq	%rax, %rcx
+	cmpq	%rsi, %rcx
+	sete	%cl
+	orb	%dl, %cl
+	jnz	2f
+
+	cmpl	$-EWOULDBLOCK, %esi
+	jne	.Lwait2
+
+	movq	$-NRW_WW, %rdx
+	addq	%rax, %rdx
+	LOCK
+	cmpxchgq %rdx, -4(%rdi)
+	jnz	.Lagain2
+
+	negl	%esi
+	movl	%esi, %eax
+	ret
+
+2:	movq	$-NRW_WW, %rdx
+	addq	%rax, %rdx
+	xorl	%ecx, %ecx
+	testq	$NRW_WP, %rax
+	cmovz	%rcx, %rsi
+	orq	%rsi, %rdx
+
+	LOCK
+	cmpxchgq %rdx, -4(%rdi)
+	jnz	.Lagain2
+
+	movl	%fs:TID, %eax
+	movl	%eax, WRITER(%rdi)
+
+	xorl	%eax, %eax
+	ret
+
+.Lovfl:	movl	$EAGAIN, %eax
+	ret
+
+.Ldeadlk:
+	movl	$EDEADLK, %eax
+	ret
+
+
+#if 0
 	pushq	%r12
 	cfi_adjust_cfa_offset(8)
 	cfi_rel_offset(%r12, 0)
@@ -263,5 +366,6 @@ pthread_rwlock_timedwrlock:
 
 19:	movl	$EINVAL, %edx
 	jmp	9b
+#endif
 	cfi_endproc
 	.size	pthread_rwlock_timedwrlock,.-pthread_rwlock_timedwrlock
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
index 03391d0..1953637 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
@@ -30,6 +30,97 @@
 	.align	16
 __pthread_rwlock_unlock:
 	cfi_startproc
+	movq	NRW_WORD(%rdi), %rax
+	movq	$NRW_WL, %rdx
+	testq	%rdx, %rax
+	jnz	.Lunlock_writer
+
+.Lagain:
+	movq	$-NRW_RC, %rdx
+	addq	%rax, %rdx
+	movq	$NRW_RC_MASK, %rcx
+	testq	%rcx, %rdx
+	jz	.Llast_reader
+
+1:	LOCK
+	cmpxchgq %rdx, NRW_WORD(%rdi)
+	jnz	.Lagain
+
+	xorl	%eax, %eax
+	ret
+
+.Llast_reader:
+	movq	$NRW_WW_MASK, %rcx
+	testq	%rax, %rcx
+	jz	1b
+
+	movq	$NRW_AR, %rcx
+	xorl	%esi, %esi
+	testq	$NRW_WP, %rax
+	cmovz	%esi, %ecx
+	orq	%rcx, %rdx
+	LOCK
+	cmpxchgq %rdx, NRW_WORD(%rdi)
+	jnz	.Lagain
+
+.Lwake_writer:
+	movl	$1, %edx
+	movl	$NRW_W_WAKEUP, %r9d
+.Lwake:	movl	$(FUTEX_WAKE_BITSET|FUTEX_PRIVATE_FLAG), %esi
+	xorl	PSHARED(%rdi), %esi
+	leaq	NRW_WORD(%rdi), %rdi
+	movl	$__NR_futex, %eax
+	syscall
+
+.Lout:	xorl	%eax, %eax
+	ret
+
+.Lunlock_writer:
+	movq	%rax, %rdx
+	movq	$NRW_WW_MASK, %rcx
+	testq	%rcx, %rax
+	jz	.Lno_writers
+	movq	$NRW_RC_MASK, %rcx
+	testq	%rcx, %rax
+	jz	2f
+	testq	$NRW_WP, %rax
+	jz	.Lwake_readers
+
+2:	movq	$~NRW_WL, %rcx
+	andq	%rcx, %rdx
+	LOCK
+	cmpxchgq %rdx, NRW_WORD(%rdi)
+	jnz	.Lunlock_writer
+	jmp	.Lwake_writer
+
+.Lno_writers:
+	movq	$~(NRW_WL|NRW_AR), %rcx
+	andq	%rcx, %rdx
+	LOCK
+	cmpxchgq %rdx, NRW_WORD(%rdi)
+	jnz	.Lunlock_writer
+
+	movq	$(NRW_RW_MASK|NRW_RC_MASK), %rcx
+	testq	%rcx, %rax
+	jz	.Lout
+
+	movl	$0x7fffffff, %edx
+	movl	$NRW_R_WAKEUP, %r9d
+	jmp	.Lwake
+
+.Lwake_readers:
+	movq	$~NRW_WL, %rcx
+	andq	%rcx, %rdx
+	LOCK
+	cmpxchgq %rdx, NRW_WORD(%rdi)
+	jnz	.Lunlock_writer
+
+	movl	$0x7fffffff, %edx
+	movl	$NRW_R_WAKEUP, %r9d
+	jmp	.Lwake
+
+
+#if 0
 	/* Get the lock.  */
 	movl	$1, %esi
 	xorl	%eax, %eax
@@ -120,6 +211,7 @@ __pthread_rwlock_unlock:
 #endif
 	callq	__lll_unlock_wake
 	jmp	8b
+#endif
 	cfi_endproc
 	.size	__pthread_rwlock_unlock,.-__pthread_rwlock_unlock
 
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
index be6b8d8..63fc2df 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
@@ -31,6 +31,94 @@
 	.align	16
 __pthread_rwlock_wrlock:
 	cfi_startproc
+	movq	NRW_WORD(%rdi), %rax
+.Lagain:
+	movq	$(NRW_WW_MASK|NRW_WL|NRW_RC_MASK), %rdx
+	testq	%rdx, %rax
+	jnz	.Lwait
+
+	// XXX Probably make AR a don't-care for !WP.  Unconditionally set it
+	movq	$(NRW_AR|NRW_WL), %rdx
+	movq	$NRW_WL, %rcx
+	testq	$NRW_WP, %rax
+	cmovz	%rcx, %rdx
+	orq	%rax, %rdx
+	LOCK
+	cmpxchgq %rdx, NRW_WORD(%rdi)
+	jnz	.Lagain
+
+	movl	%fs:TID, %eax
+	movl	%eax, WRITER(%rdi)
+
+	xorl	%eax, %eax
+	ret
+
+.Lwait:	movq	$NRW_WL, %rcx
+	testq	%rcx, %rax
+	jz	1f
+
+	movl	%fs:TID, %eax
+	cmpl	%eax, WRITER(%rdi)
+	je	.Ldeadlk
+
+1:	leaq	NRW_WORD+4(%rdi), %rdi
+
+	movq	$NRW_WW, %rdx
+	movq	$NRW_WW_MASK, %rcx
+	addq	%rax, %rdx
+	testq	%rcx, %rdx
+	jz	.Lovfl
+
+	LOCK
+	cmpxchgq  %rdx, -4(%rdi)
+	jnz	.Lagain
+
+	xorq	%r10, %r10
+	movl	$NRW_W_WAKEUP, %r9d
+.Lwait2:
+	movl	$(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi
+	movl	$__NR_futex, %eax
+	xorl	PSHARED-(NRW_WORD+4)(%rdi), %esi
+	syscall
+	movq	-4(%rdi), %rax
+
+.Lagain2:
+	movq	$(NRW_WL|NRW_RC_MASK), %rdx
+	movq	$(NRW_WL|NRW_AR), %rcx
+	testq	%rdx, %rax
+	movq	$NRW_AR, %rsi
+	setz	%cl
+	andq	%rax, %rcx
+	cmpq	%rsi, %rcx
+	sete	%cl
+	orb	%dl, %cl
+	jz	.Lwait2
+
+	movq	$-NRW_WW, %rdx
+	addq	%rax, %rdx
+	xorl	%ecx, %ecx
+	testq	$NRW_WP, %rax
+	cmovz	%rcx, %rsi
+	orq	%rsi, %rdx
+
+	LOCK
+	cmpxchgq %rdx, -4(%rdi)
+	jnz	.Lagain2
+
+	movl	%fs:TID, %eax
+	movl	%eax, WRITER(%rdi)
+
+	xorl	%eax, %eax
+	ret
+
+.Lovfl:	movl	$EAGAIN, %eax
+	ret
+
+.Ldeadlk:
+	movl	$EDEADLK, %eax
+	ret
+
+#if 0
 	xorq	%r10, %r10
 
 	/* Get the lock.  */
@@ -156,6 +244,7 @@ __pthread_rwlock_wrlock:
 	subq	$MUTEX, %rdi
 #endif
 	jmp	13b
+#endif
 	cfi_endproc
 	.size	__pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock