Fix x86, x86_64 fmax, fmin sNaN handling, add tests (bug 20947).

Various fmax and fmin function implementations mishandle sNaN
arguments:

(a) When both arguments are NaNs, the return value should be a qNaN,
but sometimes it is an sNaN if at least one argument is an sNaN.

(b) Under TS 18661-1 semantics, if either argument is an sNaN then the
result should be a qNaN (whereas if one argument is a qNaN and the
other is not a NaN, the result should be the non-NaN argument).
Various implementations treat sNaNs like qNaNs here.

This patch fixes the x86 and x86_64 versions (ignoring float and
double for 32-bit x86 given the inability to reliably avoid the sNaN
turning into a qNaN before it gets to the called function).  Tests of
sNaN inputs to these functions are added.

Note on architecture versions I haven't changed for this issue:
AArch64 already gets this right (it uses a hardware instruction with
the correct semantics for both quiet and signaling NaNs) and does not
need changes.  It's possible Alpha, IA64, SPARC might need changes
(this would be shown by the testsuite if so).

Tested for x86_64 and x86 (both i686 and i586 builds, to cover the
different x86 implementations).

	[BZ #20947]
	* sysdeps/i386/fpu/s_fmaxl.S (__fmaxl): Add the arguments when
	either is a signaling NaN.
	* sysdeps/i386/fpu/s_fminl.S (__fminl): Likewise.  Make code
	follow fmaxl more closely.
	* sysdeps/i386/i686/fpu/s_fmaxl.S (__fmaxl): Add the arguments
	when either is a signaling NaN.
	* sysdeps/i386/i686/fpu/s_fminl.S (__fminl): Likewise.
	* sysdeps/x86_64/fpu/s_fmax.S (__fmax): Likewise.
	* sysdeps/x86_64/fpu/s_fmaxf.S (__fmaxf): Likewise.
	* sysdeps/x86_64/fpu/s_fmaxl.S (__fmaxl): Likewise.
	* sysdeps/x86_64/fpu/s_fmin.S (__fmin): Likewise.
	* sysdeps/x86_64/fpu/s_fminf.S (__fminf): Likewise.
	* sysdeps/x86_64/fpu/s_fminl.S (__fminl): Likewise.
	* math/libm-test.inc (fmax_test_data): Add tests of sNaN inputs.
	(fmin_test_data): Likewise.

10 files changed, 231 insertions, 27 deletions

diff --git a/sysdeps/i386/fpu/s_fmaxl.S b/sysdeps/i386/fpu/s_fmaxl.S
index a38a194..a30401a 100644
--- a/sysdeps/i386/fpu/s_fmaxl.S
+++ b/sysdeps/i386/fpu/s_fmaxl.S
@@ -28,7 +28,13 @@ ENTRY(__fmaxl)
 
 	andb	$0x45, %ah
 	cmpb	$0x01, %ah
-	je	1f		// y == NaN
+	je	2f		// y == NaN
+
+	fxam
+	fnstsw
+	andb	$0x45, %ah
+	cmpb	$0x01, %ah
+	je	3f		// x == NaN
 
 	fucom	%st(1)
 	fnstsw
@@ -39,5 +45,27 @@ ENTRY(__fmaxl)
 1:	fstp	%st(1)
 
 	ret
+
+2:	// st(1) is a NaN; st(0) may or may not be.
+	fxam
+	fnstsw
+	andb	$0x45, %ah
+	cmpb	$0x01, %ah
+	je	4f
+	// st(1) is a NaN; st(0) is not.  Test if st(1) is signaling.
+	testb	$0x40, 23(%esp)
+	jz	4f
+	fstp	%st(1)
+	ret
+
+3:	// st(0) is a NaN; st(1) is not.  Test if st(0) is signaling.
+	testb	$0x40, 11(%esp)
+	jz	4f
+	fstp	%st(0)
+	ret
+
+4:	// Both arguments are NaNs, or one is a signaling NaN.
+	faddp
+	ret
 END(__fmaxl)
 weak_alias (__fmaxl, fmaxl)
diff --git a/sysdeps/i386/fpu/s_fminl.S b/sysdeps/i386/fpu/s_fminl.S
index fb5169b..a617e46 100644
--- a/sysdeps/i386/fpu/s_fminl.S
+++ b/sysdeps/i386/fpu/s_fminl.S
@@ -21,23 +21,51 @@
 
 	.text
 ENTRY(__fminl)
-	fldt	4(%esp)		// x
-	fldt	16(%esp)	// x : y
+	fldt	16(%esp)	// y
+	fxam
+	fnstsw
+	fldt	4(%esp)		// y : x
+
+	andb	$0x45, %ah
+	cmpb	$0x01, %ah
+	je	2f		// y == NaN
 
 	fxam
 	fnstsw
 	andb	$0x45, %ah
 	cmpb	$0x01, %ah
-	je	1f		// y == NaN
+	je	3f		// x == NaN
 
 	fucom	%st(1)
 	fnstsw
 	sahf
-	jc	2f
+	jc	1f
+
+	fxch	%st(1)
+1:	fstp	%st(1)
+
+	ret
 
-1:	fxch	%st(1)
-2:	fstp	%st(1)
+2:	// st(1) is a NaN; st(0) may or may not be.
+	fxam
+	fnstsw
+	andb	$0x45, %ah
+	cmpb	$0x01, %ah
+	je	4f
+	// st(1) is a NaN; st(0) is not.  Test if st(1) is signaling.
+	testb	$0x40, 23(%esp)
+	jz	4f
+	fstp	%st(1)
+	ret
+
+3:	// st(0) is a NaN; st(1) is not.  Test if st(0) is signaling.
+	testb	$0x40, 11(%esp)
+	jz	4f
+	fstp	%st(0)
+	ret
 
+4:	// Both arguments are NaNs, or one is a signaling NaN.
+	faddp
 	ret
 END(__fminl)
 weak_alias (__fminl, fminl)
diff --git a/sysdeps/i386/i686/fpu/s_fmaxl.S b/sysdeps/i386/i686/fpu/s_fmaxl.S
index e5dcd26..5bade5b 100644
--- a/sysdeps/i386/i686/fpu/s_fmaxl.S
+++ b/sysdeps/i386/i686/fpu/s_fmaxl.S
@@ -24,16 +24,35 @@ ENTRY(__fmaxl)
 	fldt	4(%esp)		// x
 	fldt	16(%esp)	// x : y
 
-	fucomi	%st(0), %st
-	fcmovu	%st(1), %st	// now %st contains y if not NaN, x otherwise
-
-	fxch
-
 	fucomi	%st(1), %st
+	jp	2f
 	fcmovb	%st(1), %st
 
 	fstp	%st(1)
 
 	ret
+
+2:	// Unordered.
+	fucomi	%st(0), %st
+	jp	3f
+	// st(1) is a NaN; st(0) is not.  Test if st(1) is signaling.
+	testb	$0x40, 11(%esp)
+	jz	4f
+	fstp	%st(1)
+	ret
+
+3:	// st(0) is a NaN; st(1) may or may not be.
+	fxch
+	fucomi	%st(0), %st
+	jp	4f
+	// st(1) is a NaN; st(0) is not.  Test if st(1) is signaling.
+	testb	$0x40, 23(%esp)
+	jz	4f
+	fstp	%st(1)
+	ret
+
+4:	// Both arguments are NaNs, or one is a signaling NaN.
+	faddp
+	ret
 END(__fmaxl)
 weak_alias (__fmaxl, fmaxl)
diff --git a/sysdeps/i386/i686/fpu/s_fminl.S b/sysdeps/i386/i686/fpu/s_fminl.S
index ddbd811..d586f52 100644
--- a/sysdeps/i386/i686/fpu/s_fminl.S
+++ b/sysdeps/i386/i686/fpu/s_fminl.S
@@ -24,14 +24,35 @@ ENTRY(__fminl)
 	fldt	4(%esp)		// x
 	fldt	16(%esp)	// x : y
 
-	fucomi	%st(0), %st
-	fcmovu	%st(1), %st	// now %st contains y if not NaN, x otherwise
-
 	fucomi	%st(1), %st
+	jp	2f
 	fcmovnb	%st(1), %st
 
 	fstp	%st(1)
 
 	ret
+
+2:	// Unordered.
+	fucomi	%st(0), %st
+	jp	3f
+	// st(1) is a NaN; st(0) is not.  Test if st(1) is signaling.
+	testb	$0x40, 11(%esp)
+	jz	4f
+	fstp	%st(1)
+	ret
+
+3:	// st(0) is a NaN; st(1) may or may not be.
+	fxch
+	fucomi	%st(0), %st
+	jp	4f
+	// st(1) is a NaN; st(0) is not.  Test if st(1) is signaling.
+	testb	$0x40, 23(%esp)
+	jz	4f
+	fstp	%st(1)
+	ret
+
+4:	// Both arguments are NaNs, or one is a signaling NaN.
+	faddp
+	ret
 END(__fminl)
 weak_alias (__fminl, fminl)
diff --git a/sysdeps/x86_64/fpu/s_fmax.S b/sysdeps/x86_64/fpu/s_fmax.S
index 02096c0..0ff326f 100644
--- a/sysdeps/x86_64/fpu/s_fmax.S
+++ b/sysdeps/x86_64/fpu/s_fmax.S
@@ -27,8 +27,25 @@ ENTRY(__fmax)
 	jmp	2f
 
 1:	ucomisd	%xmm1, %xmm1	// Is xmm1 a NaN?
-	jp	2f		// then return xmm0
+	jp	3f
+	// xmm0 is a NaN; xmm1 is not.  Test if xmm0 is signaling.
+	movsd	%xmm0, -8(%rsp)
+	testb	$0x8, -2(%rsp)
+	jz	4f
 	movsd	%xmm1, %xmm0	// otherwise return xmm1
+	ret
+
+3:	// xmm1 is a NaN; xmm0 may or may not be.
+	ucomisd	%xmm0, %xmm0
+	jp	4f
+	// xmm1 is a NaN; xmm0 is not.  Test if xmm1 is signaling.
+	movsd	%xmm1, -8(%rsp)
+	testb	$0x8, -2(%rsp)
+	jz	4f
+	ret
+
+4:	// Both arguments are NaNs, or one is a signaling NaN.
+	addsd	%xmm1, %xmm0
 
 2:	ret
 END(__fmax)
diff --git a/sysdeps/x86_64/fpu/s_fmaxf.S b/sysdeps/x86_64/fpu/s_fmaxf.S
index 28e1297..0f36ee0 100644
--- a/sysdeps/x86_64/fpu/s_fmaxf.S
+++ b/sysdeps/x86_64/fpu/s_fmaxf.S
@@ -27,8 +27,25 @@ ENTRY(__fmaxf)
 	jmp	2f
 
 1:	ucomiss	%xmm1, %xmm1	// Is xmm1 a NaN?
-	jp	2f		// then return xmm0
+	jp	3f
+	// xmm0 is a NaN; xmm1 is not.  Test if xmm0 is signaling.
+	movss	%xmm0, -4(%rsp)
+	testb	$0x40, -2(%rsp)
+	jz	4f
 	movss	%xmm1, %xmm0	// otherwise return xmm1
+	ret
+
+3:	// xmm1 is a NaN; xmm0 may or may not be.
+	ucomiss	%xmm0, %xmm0
+	jp	4f
+	// xmm1 is a NaN; xmm0 is not.  Test if xmm1 is signaling.
+	movss	%xmm1, -4(%rsp)
+	testb	$0x40, -2(%rsp)
+	jz	4f
+	ret
+
+4:	// Both arguments are NaNs, or one is a signaling NaN.
+	addss	%xmm1, %xmm0
 
 2:	ret
 END(__fmaxf)
diff --git a/sysdeps/x86_64/fpu/s_fmaxl.S b/sysdeps/x86_64/fpu/s_fmaxl.S
index f0c2bc0..5f0b1e0 100644
--- a/sysdeps/x86_64/fpu/s_fmaxl.S
+++ b/sysdeps/x86_64/fpu/s_fmaxl.S
@@ -24,16 +24,35 @@ ENTRY(__fmaxl)
 	fldt	8(%rsp)		// x
 	fldt	24(%rsp)	// x : y
 
-	fucomi	%st(0), %st
-	fcmovu	%st(1), %st	// now %st contains y if not NaN, x otherwise
-
-	fxch
-
 	fucomi	%st(1), %st
+	jp	2f
 	fcmovb	%st(1), %st
 
 	fstp	%st(1)
 
 	ret
+
+2:	// Unordered.
+	fucomi	%st(0), %st
+	jp	3f
+	// st(1) is a NaN; st(0) is not.  Test if st(1) is signaling.
+	testb	$0x40, 15(%rsp)
+	jz	4f
+	fstp	%st(1)
+	ret
+
+3:	// st(0) is a NaN; st(1) may or may not be.
+	fxch
+	fucomi	%st(0), %st
+	jp	4f
+	// st(1) is a NaN; st(0) is not.  Test if st(1) is signaling.
+	testb	$0x40, 31(%rsp)
+	jz	4f
+	fstp	%st(1)
+	ret
+
+4:	// Both arguments are NaNs, or one is a signaling NaN.
+	faddp
+	ret
 END(__fmaxl)
 weak_alias (__fmaxl, fmaxl)
diff --git a/sysdeps/x86_64/fpu/s_fmin.S b/sysdeps/x86_64/fpu/s_fmin.S
index fb14e2f..db89bef 100644
--- a/sysdeps/x86_64/fpu/s_fmin.S
+++ b/sysdeps/x86_64/fpu/s_fmin.S
@@ -27,8 +27,25 @@ ENTRY(__fmin)
 	jmp	2f
 
 1:	ucomisd	%xmm1, %xmm1	// Is xmm1 a NaN?
-	jp	2f		// then return xmm0
+	jp	3f
+	// xmm0 is a NaN; xmm1 is not.  Test if xmm0 is signaling.
+	movsd	%xmm0, -8(%rsp)
+	testb	$0x8, -2(%rsp)
+	jz	4f
 	movsd	%xmm1, %xmm0	// otherwise return xmm1
+	ret
+
+3:	// xmm1 is a NaN; xmm0 may or may not be.
+	ucomisd	%xmm0, %xmm0
+	jp	4f
+	// xmm1 is a NaN; xmm0 is not.  Test if xmm1 is signaling.
+	movsd	%xmm1, -8(%rsp)
+	testb	$0x8, -2(%rsp)
+	jz	4f
+	ret
+
+4:	// Both arguments are NaNs, or one is a signaling NaN.
+	addsd	%xmm1, %xmm0
 
 2:	ret
 END(__fmin)
diff --git a/sysdeps/x86_64/fpu/s_fminf.S b/sysdeps/x86_64/fpu/s_fminf.S
index c8d6d0f..41a9978 100644
--- a/sysdeps/x86_64/fpu/s_fminf.S
+++ b/sysdeps/x86_64/fpu/s_fminf.S
@@ -27,8 +27,25 @@ ENTRY(__fminf)
 	jmp	2f
 
 1:	ucomiss	%xmm1, %xmm1	// Is xmm1 a NaN?
-	jp	2f		// then return xmm0
+	jp	3f
+	// xmm0 is a NaN; xmm1 is not.  Test if xmm0 is signaling.
+	movss	%xmm0, -4(%rsp)
+	testb	$0x40, -2(%rsp)
+	jz	4f
 	movss	%xmm1, %xmm0	// otherwise return xmm1
+	ret
+
+3:	// xmm1 is a NaN; xmm0 may or may not be.
+	ucomiss	%xmm0, %xmm0
+	jp	4f
+	// xmm1 is a NaN; xmm0 is not.  Test if xmm1 is signaling.
+	movss	%xmm1, -4(%rsp)
+	testb	$0x40, -2(%rsp)
+	jz	4f
+	ret
+
+4:	// Both arguments are NaNs, or one is a signaling NaN.
+	addss	%xmm1, %xmm0
 
 2:	ret
 END(__fminf)
diff --git a/sysdeps/x86_64/fpu/s_fminl.S b/sysdeps/x86_64/fpu/s_fminl.S
index f1a06d2..12fc3fb 100644
--- a/sysdeps/x86_64/fpu/s_fminl.S
+++ b/sysdeps/x86_64/fpu/s_fminl.S
@@ -24,14 +24,35 @@ ENTRY(__fminl)
 	fldt	8(%rsp)		// x
 	fldt	24(%rsp)	// x : y
 
-	fucomi	%st(0), %st
-	fcmovu	%st(1), %st	// now %st contains y if not NaN, x otherwise
-
 	fucomi	%st(1), %st
+	jp	2f
 	fcmovnb	%st(1), %st
 
 	fstp	%st(1)
 
 	ret
+
+2:	// Unordered.
+	fucomi	%st(0), %st
+	jp	3f
+	// st(1) is a NaN; st(0) is not.  Test if st(1) is signaling.
+	testb	$0x40, 15(%rsp)
+	jz	4f
+	fstp	%st(1)
+	ret
+
+3:	// st(0) is a NaN; st(1) may or may not be.
+	fxch
+	fucomi	%st(0), %st
+	jp	4f
+	// st(1) is a NaN; st(0) is not.  Test if st(1) is signaling.
+	testb	$0x40, 31(%rsp)
+	jz	4f
+	fstp	%st(1)
+	ret
+
+4:	// Both arguments are NaNs, or one is a signaling NaN.
+	faddp
+	ret
 END(__fminl)
 weak_alias (__fminl, fminl)