Fix ldbl-128/ldbl-128ibm lgamma spurious "invalid", incorrect signgam (bug 18952).

The ldbl-128 / ldbl-128ibm implementation of lgammal converts (the
floor of minus) non-integer negative arguments to int to determine the
value of signgam.  When those values are outside the range of int,
this produces spurious "invalid" exceptions and incorrect values of
signgam.  This patch fixes this by instead determining signgam through
comparing half the integer in question to floor of half the integer.

Tested for mips64, x86_64 and x86.

	[BZ #18952]
	* sysdeps/ieee754/ldbl-128/e_lgammal_r.c (__ieee754_lgammal_r): Do
	not convert non-integer negative arguments to int to determine the
	value of signgam.
	* math/auto-libm-test-in: Add more tests of lgamma.
	* math/auto-libm-test-out: Regenerated.

5 files changed, 55 insertions, 3 deletions

diff --git a/ChangeLog b/ChangeLog
index 61a4b53..9e23da1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
 2015-09-11  Joseph Myers  <joseph@codesourcery.com>
 
+	[BZ #18952]
+	* sysdeps/ieee754/ldbl-128/e_lgammal_r.c (__ieee754_lgammal_r): Do
+	not convert non-integer negative arguments to int to determine the
+	value of signgam.
+	* math/auto-libm-test-in: Add more tests of lgamma.
+	* math/auto-libm-test-out: Regenerated.
+
 	* math/auto-libm-test-in: Add more tests of acosh, atanh, cbrt,
 	cosh, csqrt, erfc, expm1 and lgamma.
 	* math/auto-libm-test-out: Regenerated.
diff --git a/NEWS b/NEWS
index 3e78630..65e706c 100644
--- a/NEWS
+++ b/NEWS
@@ -13,7 +13,7 @@ Version 2.23
   16973, 17787, 17905, 18084, 18086, 18240, 18265, 18370, 18421, 18480,
   18525, 18610, 18618, 18647, 18661, 18674, 18675, 18681, 18757, 18778,
   18781, 18787, 18789, 18790, 18795, 18796, 18820, 18823, 18824, 18863,
-  18870, 18873, 18887, 18921.
+  18870, 18873, 18887, 18921, 18952.
 
 * The obsolete header <regexp.h> has been removed.  Programs that require
   this header must be updated to use <regex.h> instead.
diff --git a/math/auto-libm-test-in b/math/auto-libm-test-in
index 84b3df0..83eb4cf 100644
--- a/math/auto-libm-test-in
+++ b/math/auto-libm-test-in
@@ -2047,6 +2047,9 @@ lgamma -0xffffffffffffffffp-1
 lgamma -0x3ffffffffffffffffffffffffffp-1
 lgamma -0x1ffffffffffffffffffffffffffffp-1
 
+lgamma -0x100000000.8p0
+lgamma -0x100000001.8p0
+
 lgamma -0.25
 lgamma -0.5
 lgamma -0.75
diff --git a/math/auto-libm-test-out b/math/auto-libm-test-out
index 14caa1d..747815b 100644
--- a/math/auto-libm-test-out
+++ b/math/auto-libm-test-out
@@ -153351,6 +153351,48 @@ lgamma -0x1ffffffffffffffffffffffffffffp-1
 = lgamma tonearest ldbl-128 -0xf.fffffffffffffffffffffffffff8p+108L : -0x4.ca1ea7c6bcac53b28539e9281ba4p+116L 1 : inexact-ok
 = lgamma towardzero ldbl-128 -0xf.fffffffffffffffffffffffffff8p+108L : -0x4.ca1ea7c6bcac53b28539e9281ba4p+116L 1 : inexact-ok
 = lgamma upward ldbl-128 -0xf.fffffffffffffffffffffffffff8p+108L : -0x4.ca1ea7c6bcac53b28539e9281ba4p+116L 1 : inexact-ok
+lgamma -0x100000000.8p0
+= lgamma downward dbl-64 -0x1.000000008p+32 : -0x1.52e42ff102e65p+36 -1 : inexact-ok
+= lgamma tonearest dbl-64 -0x1.000000008p+32 : -0x1.52e42ff102e65p+36 -1 : inexact-ok
+= lgamma towardzero dbl-64 -0x1.000000008p+32 : -0x1.52e42ff102e64p+36 -1 : inexact-ok
+= lgamma upward dbl-64 -0x1.000000008p+32 : -0x1.52e42ff102e64p+36 -1 : inexact-ok
+= lgamma downward ldbl-96-intel -0x1.000000008p+32L : -0x1.52e42ff102e64be4p+36L -1 : inexact-ok
+= lgamma tonearest ldbl-96-intel -0x1.000000008p+32L : -0x1.52e42ff102e64be2p+36L -1 : inexact-ok
+= lgamma towardzero ldbl-96-intel -0x1.000000008p+32L : -0x1.52e42ff102e64be2p+36L -1 : inexact-ok
+= lgamma upward ldbl-96-intel -0x1.000000008p+32L : -0x1.52e42ff102e64be2p+36L -1 : inexact-ok
+= lgamma downward ldbl-96-m68k -0x1.000000008p+32L : -0x1.52e42ff102e64be4p+36L -1 : inexact-ok
+= lgamma tonearest ldbl-96-m68k -0x1.000000008p+32L : -0x1.52e42ff102e64be2p+36L -1 : inexact-ok
+= lgamma towardzero ldbl-96-m68k -0x1.000000008p+32L : -0x1.52e42ff102e64be2p+36L -1 : inexact-ok
+= lgamma upward ldbl-96-m68k -0x1.000000008p+32L : -0x1.52e42ff102e64be2p+36L -1 : inexact-ok
+= lgamma downward ldbl-128 -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d246256p+36L -1 : inexact-ok
+= lgamma tonearest ldbl-128 -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d246255p+36L -1 : inexact-ok
+= lgamma towardzero ldbl-128 -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d246255p+36L -1 : inexact-ok
+= lgamma upward ldbl-128 -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d246255p+36L -1 : inexact-ok
+= lgamma downward ldbl-128ibm -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d24628p+36L -1 : inexact-ok
+= lgamma tonearest ldbl-128ibm -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d24628p+36L -1 : inexact-ok
+= lgamma towardzero ldbl-128ibm -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d2462p+36L -1 : inexact-ok
+= lgamma upward ldbl-128ibm -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d2462p+36L -1 : inexact-ok
+lgamma -0x100000001.8p0
+= lgamma downward dbl-64 -0x1.000000018p+32 : -0x1.52e42ff265ca8p+36 1 : inexact-ok
+= lgamma tonearest dbl-64 -0x1.000000018p+32 : -0x1.52e42ff265ca8p+36 1 : inexact-ok
+= lgamma towardzero dbl-64 -0x1.000000018p+32 : -0x1.52e42ff265ca7p+36 1 : inexact-ok
+= lgamma upward dbl-64 -0x1.000000018p+32 : -0x1.52e42ff265ca7p+36 1 : inexact-ok
+= lgamma downward ldbl-96-intel -0x1.000000018p+32L : -0x1.52e42ff265ca7bd4p+36L 1 : inexact-ok
+= lgamma tonearest ldbl-96-intel -0x1.000000018p+32L : -0x1.52e42ff265ca7bd2p+36L 1 : inexact-ok
+= lgamma towardzero ldbl-96-intel -0x1.000000018p+32L : -0x1.52e42ff265ca7bd2p+36L 1 : inexact-ok
+= lgamma upward ldbl-96-intel -0x1.000000018p+32L : -0x1.52e42ff265ca7bd2p+36L 1 : inexact-ok
+= lgamma downward ldbl-96-m68k -0x1.000000018p+32L : -0x1.52e42ff265ca7bd4p+36L 1 : inexact-ok
+= lgamma tonearest ldbl-96-m68k -0x1.000000018p+32L : -0x1.52e42ff265ca7bd2p+36L 1 : inexact-ok
+= lgamma towardzero ldbl-96-m68k -0x1.000000018p+32L : -0x1.52e42ff265ca7bd2p+36L 1 : inexact-ok
+= lgamma upward ldbl-96-m68k -0x1.000000018p+32L : -0x1.52e42ff265ca7bd2p+36L 1 : inexact-ok
+= lgamma downward ldbl-128 -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be41dp+36L 1 : inexact-ok
+= lgamma tonearest ldbl-128 -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be41dp+36L 1 : inexact-ok
+= lgamma towardzero ldbl-128 -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be41cp+36L 1 : inexact-ok
+= lgamma upward ldbl-128 -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be41cp+36L 1 : inexact-ok
+= lgamma downward ldbl-128ibm -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be48p+36L 1 : inexact-ok
+= lgamma tonearest ldbl-128ibm -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be4p+36L 1 : inexact-ok
+= lgamma towardzero ldbl-128ibm -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be4p+36L 1 : inexact-ok
+= lgamma upward ldbl-128ibm -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be4p+36L 1 : inexact-ok
 lgamma -0.25
 = lgamma downward flt-32 -0x4p-4f : 0x1.96ee68p+0f -1 : inexact-ok
 = lgamma tonearest flt-32 -0x4p-4f : 0x1.96ee68p+0f -1 : inexact-ok
diff --git a/sysdeps/ieee754/ldbl-128/e_lgammal_r.c b/sysdeps/ieee754/ldbl-128/e_lgammal_r.c
index abf0f15..500aacc 100644
--- a/sysdeps/ieee754/ldbl-128/e_lgammal_r.c
+++ b/sysdeps/ieee754/ldbl-128/e_lgammal_r.c
@@ -787,8 +787,8 @@ __ieee754_lgammal_r (long double x, int *signgamp)
       p = __floorl (q);
       if (p == q)
 	return (one / (p - p));
-      i = p;
-      if ((i & 1) == 0)
+      long double halfp = p * 0.5L;
+      if (halfp == __floorl (halfp))
 	*signgamp = -1;
       else
 	*signgamp = 1;