AArch64: Add SVE memcpy

Add an initial SVE memcpy implementation.  Copies up to 32 bytes use SVE
vectors which improves the random memcpy benchmark significantly.
Cleanup the memcpy and memmove ifunc selectors.

1 files changed, 31 insertions, 20 deletions

diff --git a/sysdeps/aarch64/multiarch/memcpy.c b/sysdeps/aarch64/multiarch/memcpy.c
index a476dd548f..0486213f08 100644
--- a/sysdeps/aarch64/multiarch/memcpy.c
+++ b/sysdeps/aarch64/multiarch/memcpy.c
@@ -33,27 +33,38 @@ extern __typeof (__redirect_memcpy) __memcpy_simd attribute_hidden;
 extern __typeof (__redirect_memcpy) __memcpy_thunderx attribute_hidden;
 extern __typeof (__redirect_memcpy) __memcpy_thunderx2 attribute_hidden;
 extern __typeof (__redirect_memcpy) __memcpy_falkor attribute_hidden;
-# if HAVE_AARCH64_SVE_ASM
 extern __typeof (__redirect_memcpy) __memcpy_a64fx attribute_hidden;
-# endif
-
-libc_ifunc (__libc_memcpy,
-            (IS_THUNDERX (midr)
-	     ? __memcpy_thunderx
-	     : (IS_FALKOR (midr) || IS_PHECDA (midr)
-		? __memcpy_falkor
-		: (IS_THUNDERX2 (midr) || IS_THUNDERX2PA (midr)
-		   ? __memcpy_thunderx2
-		   : (IS_NEOVERSE_N1 (midr) || IS_NEOVERSE_N2 (midr)
-		      || IS_NEOVERSE_V1 (midr)
-		      ? __memcpy_simd
-# if HAVE_AARCH64_SVE_ASM
-		     : (IS_A64FX (midr) && sve
-			? __memcpy_a64fx
-			: __memcpy_generic))))));
-# else
-		     : __memcpy_generic)))));
-# endif
+extern __typeof (__redirect_memcpy) __memcpy_sve attribute_hidden;
+
+static inline __typeof (__redirect_memcpy) *
+select_memcpy_ifunc (void)
+{
+  INIT_ARCH ();
+
+  if (IS_NEOVERSE_N1 (midr) || IS_NEOVERSE_N2 (midr))
+    return __memcpy_simd;
+
+  if (sve && HAVE_AARCH64_SVE_ASM)
+    {
+      if (IS_A64FX (midr))
+	return __memcpy_a64fx;
+      return __memcpy_sve;
+    }
+
+  if (IS_THUNDERX (midr))
+    return __memcpy_thunderx;
+
+  if (IS_THUNDERX2 (midr) || IS_THUNDERX2PA (midr))
+    return __memcpy_thunderx2;
+
+  if (IS_FALKOR (midr) || IS_PHECDA (midr))
+    return __memcpy_falkor;
+
+  return __memcpy_generic;
+}
+
+libc_ifunc (__libc_memcpy, select_memcpy_ifunc ());
+
 # undef memcpy
 strong_alias (__libc_memcpy, memcpy);
 #endif