Redirect fma calls to __fma in libm

include/math.h has a mechanism to redirect internal calls to various
libm functions, that can often be inlined by the compiler, to call
non-exported __* names for those functions in the case when the calls
aren't inlined, with the redirection being disabled when
NO_MATH_REDIRECT.  Add fma to the functions to which this mechanism is
applied.

At present, libm-internal fma calls (generally to __builtin_fma*
functions) are only done when it's known the call will be inlined,
with alternative code not relying on an fma operation being used in
the caller otherwise.  This patch is in preparation for adding the TS
18661 / C2X narrowing fma functions to glibc; it will be natural for
the narrowing function implementations to call the underlying fma
functions unconditionally, with this either being inlined or resulting
in an __fma* call.  (Using two levels of round-to-odd computation like
that, in the case where there isn't an fma hardware instruction, isn't
optimal but is certainly a lot simpler for the initial implementation
than writing different narrowing fma implementations for all the
various pairs of formats.)

Tested with build-many-glibcs.py that installed stripped shared
libraries are unchanged by the patch (using
<https://sourceware.org/pipermail/libc-alpha/2021-September/130991.html>
to fix installed library stripping in build-many-glibcs.py).  Also
tested for x86_64.

25 files changed, 26 insertions, 0 deletions

diff --git a/include/math.h b/include/math.h
index b4772d3d3b..fa11a710a6 100644
--- a/include/math.h
+++ b/include/math.h
@@ -155,6 +155,7 @@ fabsf128 (_Float128 x)
 #   endif
 #   define MATH_REDIRECT_UNARY_ARGS(TYPE) TYPE
 #   define MATH_REDIRECT_BINARY_ARGS(TYPE) TYPE, TYPE
+#   define MATH_REDIRECT_TERNARY_ARGS(TYPE) TYPE, TYPE, TYPE
 MATH_REDIRECT (sqrt, "__ieee754_", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (ceil, "__", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (floor, "__", MATH_REDIRECT_UNARY_ARGS)
@@ -163,6 +164,7 @@ MATH_REDIRECT (rint, "__", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (trunc, "__", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (round, "__", MATH_REDIRECT_UNARY_ARGS)
 MATH_REDIRECT (copysign, "__", MATH_REDIRECT_BINARY_ARGS)
+MATH_REDIRECT (fma, "__", MATH_REDIRECT_TERNARY_ARGS)
 #  endif
 # endif
 
diff --git a/math/s_fma.c b/math/s_fma.c
index 5b0afde6b8..2dc5c5d2cb 100644
--- a/math/s_fma.c
+++ b/math/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libm-alias-double.h>
 
diff --git a/math/s_fmaf.c b/math/s_fmaf.c
index 401f0fc5ae..f1ba0a0c49 100644
--- a/math/s_fmaf.c
+++ b/math/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libm-alias-float.h>
 
diff --git a/math/s_fmal.c b/math/s_fmal.c
index 6b13ea1d5f..47a68ed235 100644
--- a/math/s_fmal.c
+++ b/math/s_fmal.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libm-alias-ldouble.h>
 
diff --git a/sysdeps/i386/i686/multiarch/s_fma.c b/sysdeps/i386/i686/multiarch/s_fma.c
index 7d66aaef7a..229f6c6526 100644
--- a/sysdeps/i386/i686/multiarch/s_fma.c
+++ b/sysdeps/i386/i686/multiarch/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <config.h>
 
 #include <math.h>
diff --git a/sysdeps/i386/i686/multiarch/s_fmaf.c b/sysdeps/i386/i686/multiarch/s_fmaf.c
index b8394d05d0..97123235f9 100644
--- a/sysdeps/i386/i686/multiarch/s_fmaf.c
+++ b/sysdeps/i386/i686/multiarch/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <config.h>
 
 #include <math.h>
diff --git a/sysdeps/ieee754/dbl-64/s_fma.c b/sysdeps/ieee754/dbl-64/s_fma.c
index 55ccf46c9a..aa4336387a 100644
--- a/sysdeps/ieee754/dbl-64/s_fma.c
+++ b/sysdeps/ieee754/dbl-64/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <float.h>
 #include <math.h>
 #include <fenv.h>
diff --git a/sysdeps/ieee754/dbl-64/s_fmaf.c b/sysdeps/ieee754/dbl-64/s_fmaf.c
index 2b4da8b763..2eb9a50d94 100644
--- a/sysdeps/ieee754/dbl-64/s_fmaf.c
+++ b/sysdeps/ieee754/dbl-64/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <fenv.h>
 #include <ieee754.h>
diff --git a/sysdeps/ieee754/float128/s_fmaf128.c b/sysdeps/ieee754/float128/s_fmaf128.c
index 6497895c8d..a900af6e3c 100644
--- a/sysdeps/ieee754/float128/s_fmaf128.c
+++ b/sysdeps/ieee754/float128/s_fmaf128.c
@@ -1,2 +1,3 @@
+#define NO_MATH_REDIRECT
 #include <float128_private.h>
 #include "../ldbl-128/s_fmal.c"
diff --git a/sysdeps/ieee754/ldbl-128/s_fma.c b/sysdeps/ieee754/ldbl-128/s_fma.c
index 47673da3d7..4795e717e8 100644
--- a/sysdeps/ieee754/ldbl-128/s_fma.c
+++ b/sysdeps/ieee754/ldbl-128/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <fenv.h>
 #include <ieee754.h>
diff --git a/sysdeps/ieee754/ldbl-128/s_fmal.c b/sysdeps/ieee754/ldbl-128/s_fmal.c
index 0f5f5f1d74..aff9efca8c 100644
--- a/sysdeps/ieee754/ldbl-128/s_fmal.c
+++ b/sysdeps/ieee754/ldbl-128/s_fmal.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <float.h>
 #include <math.h>
 #include <fenv.h>
diff --git a/sysdeps/ieee754/ldbl-128ibm/s_fmal.c b/sysdeps/ieee754/ldbl-128ibm/s_fmal.c
index f1b36c4642..a989f4cb20 100644
--- a/sysdeps/ieee754/ldbl-128ibm/s_fmal.c
+++ b/sysdeps/ieee754/ldbl-128ibm/s_fmal.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <fenv.h>
 #include <float.h>
 #include <math.h>
diff --git a/sysdeps/ieee754/ldbl-96/s_fma.c b/sysdeps/ieee754/ldbl-96/s_fma.c
index 8ad7e4dc64..417c27e534 100644
--- a/sysdeps/ieee754/ldbl-96/s_fma.c
+++ b/sysdeps/ieee754/ldbl-96/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <float.h>
 #include <math.h>
 #include <fenv.h>
diff --git a/sysdeps/ieee754/ldbl-96/s_fmal.c b/sysdeps/ieee754/ldbl-96/s_fmal.c
index fad8450b29..cd83df451b 100644
--- a/sysdeps/ieee754/ldbl-96/s_fmal.c
+++ b/sysdeps/ieee754/ldbl-96/s_fmal.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <float.h>
 #include <math.h>
 #include <fenv.h>
diff --git a/sysdeps/ieee754/soft-fp/s_fma.c b/sysdeps/ieee754/soft-fp/s_fma.c
index 9fd8ddb683..4b0d6b50aa 100644
--- a/sysdeps/ieee754/soft-fp/s_fma.c
+++ b/sysdeps/ieee754/soft-fp/s_fma.c
@@ -25,6 +25,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libc-diag.h>
 #include <libm-alias-double.h>
diff --git a/sysdeps/ieee754/soft-fp/s_fmaf.c b/sysdeps/ieee754/soft-fp/s_fmaf.c
index 2abb8c7f59..bed5990f06 100644
--- a/sysdeps/ieee754/soft-fp/s_fmaf.c
+++ b/sysdeps/ieee754/soft-fp/s_fmaf.c
@@ -25,6 +25,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libc-diag.h>
 #include <libm-alias-float.h>
diff --git a/sysdeps/ieee754/soft-fp/s_fmal.c b/sysdeps/ieee754/soft-fp/s_fmal.c
index 0a364f4652..aecec13923 100644
--- a/sysdeps/ieee754/soft-fp/s_fmal.c
+++ b/sysdeps/ieee754/soft-fp/s_fmal.c
@@ -25,6 +25,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <libc-diag.h>
 #include <libm-alias-ldouble.h>
diff --git a/sysdeps/riscv/rvd/s_fma.c b/sysdeps/riscv/rvd/s_fma.c
index 70c0cbc072..7f39cefc71 100644
--- a/sysdeps/riscv/rvd/s_fma.c
+++ b/sysdeps/riscv/rvd/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library.  If not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <fenv.h>
 #include <ieee754.h>
diff --git a/sysdeps/riscv/rvf/s_fmaf.c b/sysdeps/riscv/rvf/s_fmaf.c
index cb972fbf2f..6057a009db 100644
--- a/sysdeps/riscv/rvf/s_fmaf.c
+++ b/sysdeps/riscv/rvf/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library.  If not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <math.h>
 #include <fenv.h>
 #include <ieee754.h>
diff --git a/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fma.c b/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fma.c
index 804272ff8e..95f2cea66c 100644
--- a/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fma.c
+++ b/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fma.c
@@ -1,3 +1,4 @@
+#define NO_MATH_REDIRECT
 #include <sparc-ifunc.h>
 #include <math.h>
 #include <math_ldbl_opt.h>
diff --git a/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fmaf.c b/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fmaf.c
index e82c4a4031..cc7caa7cd2 100644
--- a/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fmaf.c
+++ b/sysdeps/sparc/sparc32/sparcv9/fpu/multiarch/s_fmaf.c
@@ -1,3 +1,4 @@
+#define NO_MATH_REDIRECT
 #include <sparc-ifunc.h>
 #include <math.h>
 #include <libm-alias-float.h>
diff --git a/sysdeps/sparc/sparc64/fpu/multiarch/s_fma.c b/sysdeps/sparc/sparc64/fpu/multiarch/s_fma.c
index 34b4f45150..44066eed46 100644
--- a/sysdeps/sparc/sparc64/fpu/multiarch/s_fma.c
+++ b/sysdeps/sparc/sparc64/fpu/multiarch/s_fma.c
@@ -1,3 +1,4 @@
+#define NO_MATH_REDIRECT
 #include <sparc-ifunc.h>
 #include <math.h>
 #include <libm-alias-double.h>
diff --git a/sysdeps/sparc/sparc64/fpu/multiarch/s_fmaf.c b/sysdeps/sparc/sparc64/fpu/multiarch/s_fmaf.c
index 08f71c8bba..3d24342454 100644
--- a/sysdeps/sparc/sparc64/fpu/multiarch/s_fmaf.c
+++ b/sysdeps/sparc/sparc64/fpu/multiarch/s_fmaf.c
@@ -1,3 +1,4 @@
+#define NO_MATH_REDIRECT
 #include <sparc-ifunc.h>
 #include <math.h>
 #include <libm-alias-float.h>
diff --git a/sysdeps/x86_64/fpu/multiarch/s_fma.c b/sysdeps/x86_64/fpu/multiarch/s_fma.c
index 8ebf72103f..89389dd77f 100644
--- a/sysdeps/x86_64/fpu/multiarch/s_fma.c
+++ b/sysdeps/x86_64/fpu/multiarch/s_fma.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <config.h>
 #include <math.h>
 #include <init-arch.h>
diff --git a/sysdeps/x86_64/fpu/multiarch/s_fmaf.c b/sysdeps/x86_64/fpu/multiarch/s_fmaf.c
index c673826563..8c1934685d 100644
--- a/sysdeps/x86_64/fpu/multiarch/s_fmaf.c
+++ b/sysdeps/x86_64/fpu/multiarch/s_fmaf.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#define NO_MATH_REDIRECT
 #include <config.h>
 #include <math.h>
 #include <init-arch.h>