Add narrowing fma functions

This patch adds the narrowing fused multiply-add functions from TS
18661-1 / TS 18661-3 / C2X to glibc's libm: ffma, ffmal, dfmal,
f32fmaf64, f32fmaf32x, f32xfmaf64 for all configurations; f32fmaf64x,
f32fmaf128, f64fmaf64x, f64fmaf128, f32xfmaf64x, f32xfmaf128,
f64xfmaf128 for configurations with _Float64x and _Float128;
__f32fmaieee128 and __f64fmaieee128 aliases in the powerpc64le case
(for calls to ffmal and dfmal when long double is IEEE binary128).
Corresponding tgmath.h macro support is also added.

The changes are mostly similar to those for the other narrowing
functions previously added, especially that for sqrt, so the
description of those generally applies to this patch as well.  As with
sqrt, I reused the same test inputs in auto-libm-test-in as for
non-narrowing fma rather than adding extra or separate inputs for
narrowing fma.  The tests in libm-test-narrow-fma.inc also follow
those for non-narrowing fma.

The non-narrowing fma has a known bug (bug 6801) that it does not set
errno on errors (overflow, underflow, Inf * 0, Inf - Inf).  Rather
than fixing this or having narrowing fma check for errors when
non-narrowing does not (complicating the cases when narrowing fma can
otherwise be an alias for a non-narrowing function), this patch does
not attempt to check for errors from narrowing fma and set errno; the
CHECK_NARROW_FMA macro is still present, but as a placeholder that
does nothing, and this missing errno setting is considered to be
covered by the existing bug rather than needing a separate open bug.
missing-errno annotations are duly added to many of the
auto-libm-test-in test inputs for fma.

This completes adding all the new functions from TS 18661-1 to glibc,
so will be followed by corresponding stdc-predef.h changes to define
__STDC_IEC_60559_BFP__ and __STDC_IEC_60559_COMPLEX__, as the support
for TS 18661-1 will be at a similar level to that for C standard
floating-point facilities up to C11 (pragmas not implemented, but
library functions done).  (There are still further changes to be done
to implement changes to the types of fromfp functions from N2548.)

Tested as followed: natively with the full glibc testsuite for x86_64
(GCC 11, 7, 6) and x86 (GCC 11); with build-many-glibcs.py with GCC
11, 7 and 6; cross testing of math/ tests for powerpc64le, powerpc32
hard float, mips64 (all three ABIs, both hard and soft float).  The
different GCC versions are to cover the different cases in tgmath.h
and tgmath.h tests properly (GCC 6 has _Float* only as typedefs in
glibc headers, GCC 7 has proper _Float* support, GCC 8 adds
__builtin_tgmath).

26 files changed, 598 insertions, 1 deletions

diff --git a/sysdeps/ieee754/dbl-64/s_f32xfmaf64.c b/sysdeps/ieee754/dbl-64/s_f32xfmaf64.c
new file mode 100644
index 0000000000..8ee50d9f4a
--- /dev/null
+++ b/sysdeps/ieee754/dbl-64/s_f32xfmaf64.c
@@ -0,0 +1 @@
+/* Defined as an alias of fma.  */
diff --git a/sysdeps/ieee754/dbl-64/s_ffma.c b/sysdeps/ieee754/dbl-64/s_ffma.c
new file mode 100644
index 0000000000..6af70d6368
--- /dev/null
+++ b/sysdeps/ieee754/dbl-64/s_ffma.c
@@ -0,0 +1,35 @@
+/* Fused multiply-add of double value, narrowing the result to float.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define f32fmaf64 __hide_f32fmaf64
+#define f32fmaf32x __hide_f32fmaf32x
+#define ffmal __hide_ffmal
+#include <math.h>
+#undef f32fmaf64
+#undef f32fmaf32x
+#undef ffmal
+
+#include <math-narrow.h>
+
+float
+__ffma (double x, double y, double z)
+{
+  NARROW_FMA_ROUND_TO_ODD (x, y, z, float, union ieee754_double, , mantissa1,
+			   false);
+}
+libm_alias_float_double (fma)
diff --git a/sysdeps/ieee754/dbl-64/s_fma.c b/sysdeps/ieee754/dbl-64/s_fma.c
index aa4336387a..87e59033e4 100644
--- a/sysdeps/ieee754/dbl-64/s_fma.c
+++ b/sysdeps/ieee754/dbl-64/s_fma.c
@@ -18,12 +18,17 @@
 
 #define NO_MATH_REDIRECT
 #include <float.h>
+#define dfmal __hide_dfmal
+#define f32xfmaf64 __hide_f32xfmaf64
 #include <math.h>
+#undef dfmal
+#undef f32xfmaf64
 #include <fenv.h>
 #include <ieee754.h>
 #include <math-barriers.h>
 #include <fenv_private.h>
 #include <libm-alias-double.h>
+#include <math-narrow-alias.h>
 #include <tininess.h>
 #include <math-use-builtins.h>
 
@@ -301,4 +306,5 @@ __fma (double x, double y, double z)
 }
 #ifndef __fma
 libm_alias_double (__fma, fma)
+libm_alias_double_narrow (__fma, fma)
 #endif
diff --git a/sysdeps/ieee754/float128/float128_private.h b/sysdeps/ieee754/float128/float128_private.h
index 13d1d63baf..a3c8d4c63a 100644
--- a/sysdeps/ieee754/float128/float128_private.h
+++ b/sysdeps/ieee754/float128/float128_private.h
@@ -285,6 +285,8 @@
 #define __daddl __f64addf128
 #define __fdivl __f32divf128
 #define __ddivl __f64divf128
+#define __ffmal __f32fmaf128
+#define __dfmal __f64fmaf128
 #define __fmull __f32mulf128
 #define __dmull __f64mulf128
 #define __fsqrtl __f32sqrtf128
diff --git a/sysdeps/ieee754/float128/s_f32fmaf128.c b/sysdeps/ieee754/float128/s_f32fmaf128.c
new file mode 100644
index 0000000000..459903db92
--- /dev/null
+++ b/sysdeps/ieee754/float128/s_f32fmaf128.c
@@ -0,0 +1,6 @@
+#define f32fmaf64x __hide_f32fmaf64x
+#define f32fmaf128 __hide_f32fmaf128
+#include <float128_private.h>
+#undef f32fmaf64x
+#undef f32fmaf128
+#include "../ldbl-128/s_ffmal.c"
diff --git a/sysdeps/ieee754/float128/s_f64fmaf128.c b/sysdeps/ieee754/float128/s_f64fmaf128.c
new file mode 100644
index 0000000000..38caab5d7d
--- /dev/null
+++ b/sysdeps/ieee754/float128/s_f64fmaf128.c
@@ -0,0 +1,10 @@
+#define f32xfmaf64x __hide_f32xfmaf64x
+#define f32xfmaf128 __hide_f32xfmaf128
+#define f64fmaf64x __hide_f64fmaf64x
+#define f64fmaf128 __hide_f64fmaf128
+#include <float128_private.h>
+#undef f32xfmaf64x
+#undef f32xfmaf128
+#undef f64fmaf64x
+#undef f64fmaf128
+#include "../ldbl-128/s_dfmal.c"
diff --git a/sysdeps/ieee754/float128/s_f64xfmaf128.c b/sysdeps/ieee754/float128/s_f64xfmaf128.c
new file mode 100644
index 0000000000..9eaec08fdd
--- /dev/null
+++ b/sysdeps/ieee754/float128/s_f64xfmaf128.c
@@ -0,0 +1,2 @@
+#include <float128_private.h>
+#include "../ldbl-128/s_f64xfmaf128.c"
diff --git a/sysdeps/ieee754/ldbl-128/s_dfmal.c b/sysdeps/ieee754/ldbl-128/s_dfmal.c
new file mode 100644
index 0000000000..e00ddba22c
--- /dev/null
+++ b/sysdeps/ieee754/ldbl-128/s_dfmal.c
@@ -0,0 +1,38 @@
+/* Fused multiply-add of long double (ldbl-128) value, narrowing the result
+   to double.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define f32xfmaf64x __hide_f32xfmaf64x
+#define f32xfmaf128 __hide_f32xfmaf128
+#define f64fmaf64x __hide_f64fmaf64x
+#define f64fmaf128 __hide_f64fmaf128
+#include <math.h>
+#undef f32xfmaf64x
+#undef f32xfmaf128
+#undef f64fmaf64x
+#undef f64fmaf128
+
+#include <math-narrow.h>
+
+double
+__dfmal (_Float128 x, _Float128 y, _Float128 z)
+{
+  NARROW_FMA_ROUND_TO_ODD (x, y, z, double, union ieee854_long_double, l,
+			   mantissa3, false);
+}
+libm_alias_double_ldouble (fma)
diff --git a/sysdeps/ieee754/ldbl-128/s_f64xfmaf128.c b/sysdeps/ieee754/ldbl-128/s_f64xfmaf128.c
new file mode 100644
index 0000000000..8616451d39
--- /dev/null
+++ b/sysdeps/ieee754/ldbl-128/s_f64xfmaf128.c
@@ -0,0 +1,39 @@
+/* Fused multiply-add of _Float128 value, converting the result to _Float64x.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <math.h>
+#include <math-narrow.h>
+#include <tininess.h>
+
+/* math_ldbl.h defines _Float128 to long double for this directory,
+   but when they are different, this function must be defined with
+   _Float128 arguments to avoid defining an alias with an incompatible
+   type.  */
+#undef _Float128
+
+#if __HAVE_FLOAT64X_LONG_DOUBLE && __HAVE_DISTINCT_FLOAT128
+_Float64x
+__f64xfmaf128 (_Float128 x, _Float128 y, _Float128 z)
+{
+  NARROW_FMA_ROUND_TO_ODD (x, y, z, _Float64x, union ieee854_long_double, l,
+			   mantissa3, TININESS_AFTER_ROUNDING);
+}
+libm_alias_float64x_float128 (fma)
+#else
+/* Defined as an alias of fmal.  */
+#endif
diff --git a/sysdeps/ieee754/ldbl-128/s_ffmal.c b/sysdeps/ieee754/ldbl-128/s_ffmal.c
new file mode 100644
index 0000000000..65d5e4830a
--- /dev/null
+++ b/sysdeps/ieee754/ldbl-128/s_ffmal.c
@@ -0,0 +1,34 @@
+/* Fused multiply-add of long double (ldbl-128) value, narrowing the result
+   to float.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define f32fmaf64x __hide_f32fmaf64x
+#define f32fmaf128 __hide_f32fmaf128
+#include <math.h>
+#undef f32fmaf64x
+#undef f32fmaf128
+
+#include <math-narrow.h>
+
+float
+__ffmal (_Float128 x, _Float128 y, _Float128 z)
+{
+  NARROW_FMA_ROUND_TO_ODD (x, y, z, float, union ieee854_long_double, l,
+			   mantissa3, false);
+}
+libm_alias_float_ldouble (fma)
diff --git a/sysdeps/ieee754/ldbl-128/s_fma.c b/sysdeps/ieee754/ldbl-128/s_fma.c
index 4795e717e8..bc90c8813f 100644
--- a/sysdeps/ieee754/ldbl-128/s_fma.c
+++ b/sysdeps/ieee754/ldbl-128/s_fma.c
@@ -17,10 +17,15 @@
    <https://www.gnu.org/licenses/>.  */
 
 #define NO_MATH_REDIRECT
+#define dfmal __hide_dfmal
+#define f32xfmaf64 __hide_f32xfmaf64
 #include <math.h>
+#undef dfmal
+#undef f32xfmaf64
 #include <fenv.h>
 #include <ieee754.h>
 #include <libm-alias-double.h>
+#include <math-narrow-alias.h>
 #include <math-use-builtins.h>
 
 /* This implementation relies on long double being more than twice as
@@ -58,4 +63,5 @@ __fma (double x, double y, double z)
 }
 #ifndef __fma
 libm_alias_double (__fma, fma)
+libm_alias_double_narrow (__fma, fma)
 #endif
diff --git a/sysdeps/ieee754/ldbl-128/s_fmal.c b/sysdeps/ieee754/ldbl-128/s_fmal.c
index aff9efca8c..4595ad2bc5 100644
--- a/sysdeps/ieee754/ldbl-128/s_fmal.c
+++ b/sysdeps/ieee754/ldbl-128/s_fmal.c
@@ -18,12 +18,15 @@
 
 #define NO_MATH_REDIRECT
 #include <float.h>
+#define f64xfmaf128 __hide_f64xfmaf128
 #include <math.h>
+#undef f64xfmaf128
 #include <fenv.h>
 #include <ieee754.h>
 #include <math-barriers.h>
 #include <math_private.h>
 #include <libm-alias-ldouble.h>
+#include <math-narrow-alias.h>
 #include <tininess.h>
 #include <math-use-builtins.h>
 
@@ -303,3 +306,4 @@ __fmal (_Float128 x, _Float128 y, _Float128 z)
 #endif /* ! USE_FMAL_BUILTIN  */
 }
 libm_alias_ldouble (__fma, fma)
+libm_alias_ldouble_narrow (__fma, fma)
diff --git a/sysdeps/ieee754/ldbl-128ibm-compat/Versions b/sysdeps/ieee754/ldbl-128ibm-compat/Versions
index 885ffb977f..03a66134c4 100644
--- a/sysdeps/ieee754/ldbl-128ibm-compat/Versions
+++ b/sysdeps/ieee754/ldbl-128ibm-compat/Versions
@@ -124,6 +124,8 @@ libm {
     __ynieee128;
   }
   GLIBC_2.35 {
+    __f32fmaieee128;
+    __f64fmaieee128;
     __f32sqrtieee128;
     __f64sqrtieee128;
   }
diff --git a/sysdeps/ieee754/ldbl-128ibm/s_dfmal.c b/sysdeps/ieee754/ldbl-128ibm/s_dfmal.c
new file mode 100644
index 0000000000..644fe413d1
--- /dev/null
+++ b/sysdeps/ieee754/ldbl-128ibm/s_dfmal.c
@@ -0,0 +1,28 @@
+/* Fused multiply-add of long double (ldbl-128ibm) value, narrowing the result
+   to double.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <math.h>
+#include <math-narrow.h>
+
+double
+__dfmal (long double x, long double y, long double z)
+{
+  NARROW_FMA_TRIVIAL (x, y, z, double, l);
+}
+libm_alias_double_ldouble (fma)
diff --git a/sysdeps/ieee754/ldbl-128ibm/s_ffmal.c b/sysdeps/ieee754/ldbl-128ibm/s_ffmal.c
new file mode 100644
index 0000000000..e71c40c80a
--- /dev/null
+++ b/sysdeps/ieee754/ldbl-128ibm/s_ffmal.c
@@ -0,0 +1,28 @@
+/* Fused multiply-add of long double (ldbl-128ibm) value, narrowing the result
+   to float.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <math.h>
+#include <math-narrow.h>
+
+float
+__ffmal (long double x, long double y, long double z)
+{
+  NARROW_FMA_TRIVIAL (x, y, z, float, l);
+}
+libm_alias_float_ldouble (fma)
diff --git a/sysdeps/ieee754/ldbl-96/s_dfmal.c b/sysdeps/ieee754/ldbl-96/s_dfmal.c
new file mode 100644
index 0000000000..5a1ecf7e79
--- /dev/null
+++ b/sysdeps/ieee754/ldbl-96/s_dfmal.c
@@ -0,0 +1,34 @@
+/* Fused multiply-add of long double (ldbl-96) value, narrowing the result
+   to double.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define f32xfmaf64x __hide_f32xfmaf64x
+#define f64fmaf64x __hide_f64fmaf64x
+#include <math.h>
+#undef f32xfmaf64x
+#undef f64fmaf64x
+
+#include <math-narrow.h>
+
+double
+__dfmal (long double x, long double y, long double z)
+{
+  NARROW_FMA_ROUND_TO_ODD (x, y, z, double, union ieee854_long_double, l,
+			   mantissa1, false);
+}
+libm_alias_double_ldouble (fma)
diff --git a/sysdeps/ieee754/ldbl-96/s_ffmal.c b/sysdeps/ieee754/ldbl-96/s_ffmal.c
new file mode 100644
index 0000000000..71f96b880b
--- /dev/null
+++ b/sysdeps/ieee754/ldbl-96/s_ffmal.c
@@ -0,0 +1,32 @@
+/* Fused multiply-add of long double (ldbl-96) value, narrowing the result
+   to float.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define f32fmaf64x __hide_f32fmaf64x
+#include <math.h>
+#undef f32fmaf64x
+
+#include <math-narrow.h>
+
+float
+__ffmal (long double x, long double y, long double z)
+{
+  NARROW_FMA_ROUND_TO_ODD (x, y, z, float, union ieee854_long_double, l,
+			   mantissa1, false);
+}
+libm_alias_float_ldouble (fma)
diff --git a/sysdeps/ieee754/ldbl-96/s_fma.c b/sysdeps/ieee754/ldbl-96/s_fma.c
index 417c27e534..025c60651e 100644
--- a/sysdeps/ieee754/ldbl-96/s_fma.c
+++ b/sysdeps/ieee754/ldbl-96/s_fma.c
@@ -18,11 +18,16 @@
 
 #define NO_MATH_REDIRECT
 #include <float.h>
+#define dfmal __hide_dfmal
+#define f32xfmaf64 __hide_f32xfmaf64
 #include <math.h>
+#undef dfmal
+#undef f32xfmaf64
 #include <fenv.h>
 #include <ieee754.h>
 #include <math-barriers.h>
 #include <libm-alias-double.h>
+#include <math-narrow-alias.h>
 
 /* This implementation uses rounding to odd to avoid problems with
    double rounding.  See a paper by Boldo and Melquiond:
@@ -97,4 +102,5 @@ __fma (double x, double y, double z)
 }
 #ifndef __fma
 libm_alias_double (__fma, fma)
+libm_alias_double_narrow (__fma, fma)
 #endif
diff --git a/sysdeps/ieee754/ldbl-opt/Makefile b/sysdeps/ieee754/ldbl-opt/Makefile
index dff53cc0d8..6b21680033 100644
--- a/sysdeps/ieee754/ldbl-opt/Makefile
+++ b/sysdeps/ieee754/ldbl-opt/Makefile
@@ -45,7 +45,7 @@ libnldbl-calls = asprintf dprintf fprintf fscanf fwprintf fwscanf iovfscanf \
 		 nextup nextdown totalorder totalordermag getpayload \
 		 canonicalize setpayload setpayloadsig llogb fmaxmag fminmag \
 		 roundeven fromfp ufromfp fromfpx ufromfpx fadd dadd \
-		 fdiv ddiv fmul dmul fsqrt dsqrt fsub dsub
+		 fdiv ddiv ffma dfma fmul dmul fsqrt dsqrt fsub dsub
 libnldbl-routines = $(libnldbl-calls:%=nldbl-%)
 libnldbl-inhibit-o = $(object-suffixes)
 libnldbl-static-only-routines = $(libnldbl-routines)
@@ -89,6 +89,7 @@ CFLAGS-nldbl-ctan.c = -fno-builtin-ctanl
 CFLAGS-nldbl-ctanh.c = -fno-builtin-ctanhl
 CFLAGS-nldbl-dadd.c = -fno-builtin-daddl
 CFLAGS-nldbl-ddiv.c = -fno-builtin-ddivl
+CFLAGS-nldbl-dfma.c = -fno-builtin-dfmal
 CFLAGS-nldbl-dmul.c = -fno-builtin-dmull
 CFLAGS-nldbl-dsqrt.c = -fno-builtin-dsqrtl
 CFLAGS-nldbl-dsub.c = -fno-builtin-dsubl
@@ -102,6 +103,7 @@ CFLAGS-nldbl-fabs.c = -fno-builtin-fabsl
 CFLAGS-nldbl-fadd.c = -fno-builtin-faddl
 CFLAGS-nldbl-fdim.c = -fno-builtin-fdiml
 CFLAGS-nldbl-fdiv.c = -fno-builtin-fdivl
+CFLAGS-nldbl-ffma.c = -fno-builtin-ffmal
 CFLAGS-nldbl-finite.c = -fno-builtin-finitel
 CFLAGS-nldbl-floor.c = -fno-builtin-floorl
 CFLAGS-nldbl-fma.c = -fno-builtin-fmal
diff --git a/sysdeps/ieee754/ldbl-opt/nldbl-dfma.c b/sysdeps/ieee754/ldbl-opt/nldbl-dfma.c
new file mode 100644
index 0000000000..291f3e092f
--- /dev/null
+++ b/sysdeps/ieee754/ldbl-opt/nldbl-dfma.c
@@ -0,0 +1,28 @@
+/* Compatibility routine for IEEE double as long double for dfma.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define dfmal __hide_dfmal
+#include "nldbl-compat.h"
+#undef dfmal
+
+double
+attribute_hidden
+dfmal (double x, double y, double z)
+{
+  return fma (x, y, z);
+}
diff --git a/sysdeps/ieee754/ldbl-opt/nldbl-ffma.c b/sysdeps/ieee754/ldbl-opt/nldbl-ffma.c
new file mode 100644
index 0000000000..feabea512e
--- /dev/null
+++ b/sysdeps/ieee754/ldbl-opt/nldbl-ffma.c
@@ -0,0 +1,28 @@
+/* Compatibility routine for IEEE double as long double for ffma.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define ffmal __hide_ffmal
+#include "nldbl-compat.h"
+#undef ffmal
+
+float
+attribute_hidden
+ffmal (double x, double y, double z)
+{
+  return ffma (x, y, z);
+}
diff --git a/sysdeps/ieee754/soft-fp/s_dfmal.c b/sysdeps/ieee754/soft-fp/s_dfmal.c
new file mode 100644
index 0000000000..6e8b081a9e
--- /dev/null
+++ b/sysdeps/ieee754/soft-fp/s_dfmal.c
@@ -0,0 +1,74 @@
+/* Fused multiply-add of long double (ldbl-128) values, narrowing the result to
+   double, using soft-fp.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define f32xfmaf64x __hide_f32xfmaf64x
+#define f32xfmaf128 __hide_f32xfmaf128
+#define f64fmaf64x __hide_f64fmaf64x
+#define f64fmaf128 __hide_f64fmaf128
+#include <math.h>
+#undef f32xfmaf64x
+#undef f32xfmaf128
+#undef f64fmaf64x
+#undef f64fmaf128
+
+#include <math-narrow.h>
+#include <libc-diag.h>
+
+/* R_e is not set in cases where it is not used in packing, but the
+   compiler does not see that it is set in all cases where it is
+   used, resulting in warnings that it may be used uninitialized.
+   The location of the warning differs in different versions of GCC,
+   it may be where R is defined using a macro or it may be where the
+   macro is defined.  */
+DIAG_PUSH_NEEDS_COMMENT;
+DIAG_IGNORE_NEEDS_COMMENT (4.9, "-Wmaybe-uninitialized");
+
+#include <soft-fp.h>
+#include <double.h>
+#include <quad.h>
+
+double
+__dfmal (_Float128 x, _Float128 y, _Float128 z)
+{
+  FP_DECL_EX;
+  FP_DECL_Q (X);
+  FP_DECL_Q (Y);
+  FP_DECL_Q (Z);
+  FP_DECL_Q (R);
+  FP_DECL_D (RN);
+  double ret;
+
+  FP_INIT_ROUNDMODE;
+  FP_UNPACK_Q (X, x);
+  FP_UNPACK_Q (Y, y);
+  FP_UNPACK_Q (Z, z);
+  FP_FMA_Q (R, X, Y, Z);
+#if _FP_W_TYPE_SIZE < 64
+  FP_TRUNC_COOKED (D, Q, 2, 4, RN, R);
+#else
+  FP_TRUNC_COOKED (D, Q, 1, 2, RN, R);
+#endif
+  FP_PACK_D (ret, RN);
+  FP_HANDLE_EXCEPTIONS;
+  CHECK_NARROW_FMA (ret, x, y, z);
+  return ret;
+}
+DIAG_POP_NEEDS_COMMENT;
+
+libm_alias_double_ldouble (fma)
diff --git a/sysdeps/ieee754/soft-fp/s_ffma.c b/sysdeps/ieee754/soft-fp/s_ffma.c
new file mode 100644
index 0000000000..3ab5844b0b
--- /dev/null
+++ b/sysdeps/ieee754/soft-fp/s_ffma.c
@@ -0,0 +1,72 @@
+/* Fused multiply-add of double values, narrowing the result to float,
+   using soft-fp.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define f32fmaf64 __hide_f32fmaf64
+#define f32fmaf32x __hide_f32fmaf32x
+#define ffmal __hide_ffmal
+#include <math.h>
+#undef f32fmaf64
+#undef f32fmaf32x
+#undef ffmal
+
+#include <math-narrow.h>
+#include <libc-diag.h>
+
+/* R_e is not set in cases where it is not used in packing, but the
+   compiler does not see that it is set in all cases where it is
+   used, resulting in warnings that it may be used uninitialized.
+   The location of the warning differs in different versions of GCC,
+   it may be where R is defined using a macro or it may be where the
+   macro is defined.  */
+DIAG_PUSH_NEEDS_COMMENT;
+DIAG_IGNORE_NEEDS_COMMENT (4.9, "-Wmaybe-uninitialized");
+
+#include <soft-fp.h>
+#include <single.h>
+#include <double.h>
+
+float
+__ffma (double x, double y, double z)
+{
+  FP_DECL_EX;
+  FP_DECL_D (X);
+  FP_DECL_D (Y);
+  FP_DECL_D (Z);
+  FP_DECL_D (R);
+  FP_DECL_S (RN);
+  float ret;
+
+  FP_INIT_ROUNDMODE;
+  FP_UNPACK_D (X, x);
+  FP_UNPACK_D (Y, y);
+  FP_UNPACK_D (Z, z);
+  FP_FMA_D (R, X, Y, Z);
+#if _FP_W_TYPE_SIZE < _FP_FRACBITS_D
+  FP_TRUNC_COOKED (S, D, 1, 2, RN, R);
+#else
+  FP_TRUNC_COOKED (S, D, 1, 1, RN, R);
+#endif
+  FP_PACK_S (ret, RN);
+  FP_HANDLE_EXCEPTIONS;
+  CHECK_NARROW_FMA (ret, x, y, z);
+  return ret;
+}
+DIAG_POP_NEEDS_COMMENT;
+
+libm_alias_float_double (fma)
diff --git a/sysdeps/ieee754/soft-fp/s_ffmal.c b/sysdeps/ieee754/soft-fp/s_ffmal.c
new file mode 100644
index 0000000000..15e537e978
--- /dev/null
+++ b/sysdeps/ieee754/soft-fp/s_ffmal.c
@@ -0,0 +1,70 @@
+/* Fused multiply-add of long double (ldbl-128) values, narrowing the result to
+   float, using soft-fp.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define f32fmaf64x __hide_f32fmaf64x
+#define f32fmaf128 __hide_f32fmaf128
+#include <math.h>
+#undef f32fmaf64x
+#undef f32fmaf128
+
+#include <math-narrow.h>
+#include <libc-diag.h>
+
+/* R_e is not set in cases where it is not used in packing, but the
+   compiler does not see that it is set in all cases where it is
+   used, resulting in warnings that it may be used uninitialized.
+   The location of the warning differs in different versions of GCC,
+   it may be where R is defined using a macro or it may be where the
+   macro is defined.  */
+DIAG_PUSH_NEEDS_COMMENT;
+DIAG_IGNORE_NEEDS_COMMENT (4.9, "-Wmaybe-uninitialized");
+
+#include <soft-fp.h>
+#include <single.h>
+#include <quad.h>
+
+float
+__ffmal (_Float128 x, _Float128 y, _Float128 z)
+{
+  FP_DECL_EX;
+  FP_DECL_Q (X);
+  FP_DECL_Q (Y);
+  FP_DECL_Q (Z);
+  FP_DECL_Q (R);
+  FP_DECL_S (RN);
+  float ret;
+
+  FP_INIT_ROUNDMODE;
+  FP_UNPACK_Q (X, x);
+  FP_UNPACK_Q (Y, y);
+  FP_UNPACK_Q (Z, z);
+  FP_FMA_Q (R, X, Y, Z);
+#if _FP_W_TYPE_SIZE < 64
+  FP_TRUNC_COOKED (S, Q, 1, 4, RN, R);
+#else
+  FP_TRUNC_COOKED (S, Q, 1, 2, RN, R);
+#endif
+  FP_PACK_S (ret, RN);
+  FP_HANDLE_EXCEPTIONS;
+  CHECK_NARROW_FMA (ret, x, y, z);
+  return ret;
+}
+DIAG_POP_NEEDS_COMMENT;
+
+libm_alias_float_ldouble (fma)
diff --git a/sysdeps/ieee754/soft-fp/s_fma.c b/sysdeps/ieee754/soft-fp/s_fma.c
index 4b0d6b50aa..d3870452dd 100644
--- a/sysdeps/ieee754/soft-fp/s_fma.c
+++ b/sysdeps/ieee754/soft-fp/s_fma.c
@@ -26,9 +26,14 @@
    <https://www.gnu.org/licenses/>.  */
 
 #define NO_MATH_REDIRECT
+#define dfmal __hide_dfmal
+#define f32xfmaf64 __hide_f32xfmaf64
 #include <math.h>
+#undef dfmal
+#undef f32xfmaf64
 #include <libc-diag.h>
 #include <libm-alias-double.h>
+#include <math-narrow-alias.h>
 
 /* R_e is not set in cases where it is not used in packing, but the
    compiler does not see that it is set in all cases where it is
@@ -66,4 +71,5 @@ DIAG_POP_NEEDS_COMMENT;
 
 #ifndef __fma
 libm_alias_double (__fma, fma)
+libm_alias_double_narrow (__fma, fma)
 #endif
diff --git a/sysdeps/ieee754/soft-fp/s_fmal.c b/sysdeps/ieee754/soft-fp/s_fmal.c
index aecec13923..7be917a46e 100644
--- a/sysdeps/ieee754/soft-fp/s_fmal.c
+++ b/sysdeps/ieee754/soft-fp/s_fmal.c
@@ -26,9 +26,12 @@
    <https://www.gnu.org/licenses/>.  */
 
 #define NO_MATH_REDIRECT
+#define f64xfmaf128 __hide_f64xfmaf128
 #include <math.h>
+#undef f64xfmaf128
 #include <libc-diag.h>
 #include <libm-alias-ldouble.h>
+#include <math-narrow-alias.h>
 
 /* R_e is not set in cases where it is not used in packing, but the
    compiler does not see that it is set in all cases where it is
@@ -65,3 +68,4 @@ __fmal (long double a, long double b, long double c)
 DIAG_POP_NEEDS_COMMENT;
 
 libm_alias_ldouble (__fma, fma)
+libm_alias_ldouble_narrow (__fma, fma)