Redirect fma calls to __fma in libm

include/math.h has a mechanism to redirect internal calls to various
libm functions, that can often be inlined by the compiler, to call
non-exported __* names for those functions in the case when the calls
aren't inlined, with the redirection being disabled when
NO_MATH_REDIRECT.  Add fma to the functions to which this mechanism is
applied.

At present, libm-internal fma calls (generally to __builtin_fma*
functions) are only done when it's known the call will be inlined,
with alternative code not relying on an fma operation being used in
the caller otherwise.  This patch is in preparation for adding the TS
18661 / C2X narrowing fma functions to glibc; it will be natural for
the narrowing function implementations to call the underlying fma
functions unconditionally, with this either being inlined or resulting
in an __fma* call.  (Using two levels of round-to-odd computation like
that, in the case where there isn't an fma hardware instruction, isn't
optimal but is certainly a lot simpler for the initial implementation
than writing different narrowing fma implementations for all the
various pairs of formats.)

Tested with build-many-glibcs.py that installed stripped shared
libraries are unchanged by the patch (using
<https://sourceware.org/pipermail/libc-alpha/2021-September/130991.html>
to fix installed library stripping in build-many-glibcs.py).  Also
tested for x86_64.

25 files changed, 26 insertions, 0 deletions

diff --git a/include/math.h b/include/math.h
index b4772d3..fa11a71 100644
--- a/include/math.h
+++ b/include/math.h
@@ -155,6 +155,7 @@ fabsf128 (_Float128 x)
 #   endif
 #   define MATH_REDIRECT_UNARY_ARGS(TYPE) TYPE
 #   define MATH_REDIRECT_BINARY_ARGS(TYPE) TYPE, TYPE
+#   define MATH_REDIRECT_TERNARY_ARGS(TYPE) TYPE, TYPE, TYPE
 MATH_REDIRECT (sqrt, "__ieee754_", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (ceil, "__", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (floor, "__", MATH_REDIRECT_UNARY_ARGS)
@@ -163,6 +164,7 @@ MATH_REDIRECT (rint, "__", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (trunc, "__", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (round, "__", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (copysign, "__", MATH_REDIRECT_BINARY_ARGS)
+MATH_REDIRECT (fma, "__", MATH_REDIRECT_TERNARY_ARGS)
 #  endif
 # endif
 
diff --git a/math/s_fma.c b/math/s_fma.c
index 5b0afde..2dc5c5d 100644
--- a/math/s_fma.c
+++ b/math/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libm-alias-double.h>
 
diff --git a/math/s_fmaf.c b/math/s_fmaf.c
index 401f0fc..f1ba0a0 100644
--- a/math/s_fmaf.c
+++ b/math/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libm-alias-float.h>
 
diff --git a/math/s_fmal.c b/math/s_fmal.c
index 6b13ea1d..47a68ed 100644
--- a/math/s_fmal.c
+++ b/math/s_fmal.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libm-alias-ldouble.h>
 
diff --git a/sysdeps/i386/i686/multiarch/s_fma.c b/sysdeps/i386/i686/multiarch/s_fma.c
index 7d66aae..229f6c6 100644
--- a/sysdeps/i386/i686/multiarch/s_fma.c
+++ b/sysdeps/i386/i686/multiarch/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <config.h>
 
 #include <math.h>
diff --git a/sysdeps/i386/i686/multiarch/s_fmaf.c b/sysdeps/i386/i686/multiarch/s_fmaf.c
index b8394d0..9712323 100644
--- a/sysdeps/i386/i686/multiarch/s_fmaf.c
+++ b/sysdeps/i386/i686/multiarch/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <config.h>
 
 #include <math.h>
diff --git a/sysdeps/ieee754/dbl-64/s_fma.c b/sysdeps/ieee754/dbl-64/s_fma.c
index 55ccf46..aa43363 100644
--- a/sysdeps/ieee754/dbl-64/s_fma.c
+++ b/sysdeps/ieee754/dbl-64/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <float.h>
 #include <math.h>
 #include <fenv.h>
diff --git a/sysdeps/ieee754/dbl-64/s_fmaf.c b/sysdeps/ieee754/dbl-64/s_fmaf.c
index 2b4da8b..2eb9a50 100644
--- a/sysdeps/ieee754/dbl-64/s_fmaf.c
+++ b/sysdeps/ieee754/dbl-64/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <fenv.h>
 #include <ieee754.h>
diff --git a/sysdeps/ieee754/float128/s_fmaf128.c b/sysdeps/ieee754/float128/s_fmaf128.c
index 6497895..a900af6 100644
--- a/sysdeps/ieee754/float128/s_fmaf128.c
+++ b/sysdeps/ieee754/float128/s_fmaf128.c
@@ -1,2 +1,3 @@
+#define NO_MATH_REDIRECT
 #include <float128_private.h>
 #include "../ldbl-128/s_fmal.c"
diff --git a/sysdeps/ieee754/ldbl-128/s_fma.c b/sysdeps/ieee754/ldbl-128/s_fma.c
index 47673da..4795e71 100644
--- a/sysdeps/ieee754/ldbl-128/s_fma.c
+++ b/sysdeps/ieee754/ldbl-128/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <fenv.h>
 #include <ieee754.h>
diff --git a/sysdeps/ieee754/ldbl-128/s_fmal.c b/sysdeps/ieee754/ldbl-128/s_fmal.c
index 0f5f5f1..aff9efc 100644
--- a/sysdeps/ieee754/ldbl-128/s_fmal.c
+++ b/sysdeps/ieee754/ldbl-128/s_fmal.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <float.h>
 #include <math.h>
 #include <fenv.h>
diff --git a/sysdeps/ieee754/ldbl-128ibm/s_fmal.c b/sysdeps/ieee754/ldbl-128ibm/s_fmal.c
index f1b36c4..a989f4c 100644
--- a/sysdeps/ieee754/ldbl-128ibm/s_fmal.c
+++ b/sysdeps/ieee754/ldbl-128ibm/s_fmal.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <fenv.h>
 #include <float.h>
 #include <math.h>
diff --git a/sysdeps/ieee754/ldbl-96/s_fma.c b/sysdeps/ieee754/ldbl-96/s_fma.c
index 8ad7e4d..417c27e 100644
--- a/sysdeps/ieee754/ldbl-96/s_fma.c
+++ b/sysdeps/ieee754/ldbl-96/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <float.h>
 #include <math.h>
 #include <fenv.h>
diff --git a/sysdeps/ieee754/ldbl-96/s_fmal.c b/sysdeps/ieee754/ldbl-96/s_fmal.c
index fad8450..cd83df4 100644
--- a/sysdeps/ieee754/ldbl-96/s_fmal.c
+++ b/sysdeps/ieee754/ldbl-96/s_fmal.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <float.h>
 #include <math.h>
 #include <fenv.h>
diff --git a/sysdeps/ieee754/soft-fp/s_fma.c b/sysdeps/ieee754/soft-fp/s_fma.c
index 9fd8ddb..4b0d6b5 100644
--- a/sysdeps/ieee754/soft-fp/s_fma.c
+++ b/sysdeps/ieee754/soft-fp/s_fma.c
@@ -25,6 +25,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libc-diag.h>
 #include <libm-alias-double.h>
diff --git a/sysdeps/ieee754/soft-fp/s_fmaf.c b/sysdeps/ieee754/soft-fp/s_fmaf.c
index 2abb8c7..bed5990 100644
--- a/sysdeps/ieee754/soft-fp/s_fmaf.c
+++ b/sysdeps/ieee754/soft-fp/s_fmaf.c
@@ -25,6 +25,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libc-diag.h>
 #include <libm-alias-float.h>
diff --git a/sysdeps/ieee754/soft-fp/s_fmal.c b/sysdeps/ieee754/soft-fp/s_fmal.c
index 0a364f4..aecec13 100644
--- a/sysdeps/ieee754/soft-fp/s_fmal.c
+++ b/sysdeps/ieee754/soft-fp/s_fmal.c
@@ -25,6 +25,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libc-diag.h>
 #include <libm-alias-ldouble.h>
diff --git a/sysdeps/riscv/rvd/s_fma.c b/sysdeps/riscv/rvd/s_fma.c
index 70c0cbc..7f39cef 100644
--- a/sysdeps/riscv/rvd/s_fma.c
+++ b/sysdeps/riscv/rvd/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library.  If not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <fenv.h>
 #include <ieee754.h>
diff --git a/sysdeps/riscv/rvf/s_fmaf.c b/sysdeps/riscv/rvf/s_fmaf.c
index cb972fb..6057a00 100644
--- a/sysdeps/riscv/rvf/s_fmaf.c
+++ b/sysdeps/riscv/rvf/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library.  If not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <fenv.h>
 #include <ieee754.h>
diff --git a/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fma.c b/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fma.c
index 804272f..95f2cea 100644
--- a/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fma.c
+++ b/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fma.c
@@ -1,3 +1,4 @@
+#define NO_MATH_REDIRECT
 #include <sparc-ifunc.h>
 #include <math.h>
 #include <math_ldbl_opt.h>
diff --git a/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fmaf.c b/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fmaf.c
index e82c4a4..cc7caa7 100644
--- a/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fmaf.c
+++ b/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fmaf.c
@@ -1,3 +1,4 @@
+#define NO_MATH_REDIRECT
 #include <sparc-ifunc.h>
 #include <math.h>
 #include <libm-alias-float.h>
diff --git a/sysdeps/sparc/sparc64/fpu/multiarch/s_fma.c b/sysdeps/sparc/sparc64/fpu/multiarch/s_fma.c
index 34b4f45..44066ee 100644
--- a/sysdeps/sparc/sparc64/fpu/multiarch/s_fma.c
+++ b/sysdeps/sparc/sparc64/fpu/multiarch/s_fma.c
@@ -1,3 +1,4 @@
+#define NO_MATH_REDIRECT
 #include <sparc-ifunc.h>
 #include <math.h>
 #include <libm-alias-double.h>
diff --git a/sysdeps/sparc/sparc64/fpu/multiarch/s_fmaf.c b/sysdeps/sparc/sparc64/fpu/multiarch/s_fmaf.c
index 08f71c8..3d24342 100644
--- a/sysdeps/sparc/sparc64/fpu/multiarch/s_fmaf.c
+++ b/sysdeps/sparc/sparc64/fpu/multiarch/s_fmaf.c
@@ -1,3 +1,4 @@
+#define NO_MATH_REDIRECT
 #include <sparc-ifunc.h>
 #include <math.h>
 #include <libm-alias-float.h>
diff --git a/sysdeps/x86_64/fpu/multiarch/s_fma.c b/sysdeps/x86_64/fpu/multiarch/s_fma.c
index 8ebf721..89389dd 100644
--- a/sysdeps/x86_64/fpu/multiarch/s_fma.c
+++ b/sysdeps/x86_64/fpu/multiarch/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <config.h>
 #include <math.h>
 #include <init-arch.h>
diff --git a/sysdeps/x86_64/fpu/multiarch/s_fmaf.c b/sysdeps/x86_64/fpu/multiarch/s_fmaf.c
index c673826..8c19346 100644
--- a/sysdeps/x86_64/fpu/multiarch/s_fmaf.c
+++ b/sysdeps/x86_64/fpu/multiarch/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <config.h>
 #include <math.h>
 #include <init-arch.h>