AArch64: Add SVE memcpy

Add an initial SVE memcpy implementation.  Copies up to 32 bytes use SVE
vectors which improves the random memcpy benchmark significantly.
Cleanup the memcpy and memmove ifunc selectors.

1 files changed, 31 insertions, 20 deletions

diff --git a/sysdeps/aarch64/multiarch/memmove.c b/sysdeps/aarch64/multiarch/memmove.c
index 4f7d7eedfd..261996ecc4 100644
--- a/sysdeps/aarch64/multiarch/memmove.c
+++ b/sysdeps/aarch64/multiarch/memmove.c
@@ -33,27 +33,38 @@ extern __typeof (__redirect_memmove) __memmove_simd attribute_hidden;
 extern __typeof (__redirect_memmove) __memmove_thunderx attribute_hidden;
 extern __typeof (__redirect_memmove) __memmove_thunderx2 attribute_hidden;
 extern __typeof (__redirect_memmove) __memmove_falkor attribute_hidden;
-# if HAVE_AARCH64_SVE_ASM
 extern __typeof (__redirect_memmove) __memmove_a64fx attribute_hidden;
-# endif
-
-libc_ifunc (__libc_memmove,
-            (IS_THUNDERX (midr)
-	     ? __memmove_thunderx
-	     : (IS_FALKOR (midr) || IS_PHECDA (midr)
-		? __memmove_falkor
-		: (IS_THUNDERX2 (midr) || IS_THUNDERX2PA (midr)
-		   ? __memmove_thunderx2
-		   : (IS_NEOVERSE_N1 (midr) || IS_NEOVERSE_N2 (midr)
-		      || IS_NEOVERSE_V1 (midr)
-		      ? __memmove_simd
-# if HAVE_AARCH64_SVE_ASM
-		     : (IS_A64FX (midr) && sve
-			? __memmove_a64fx
-			: __memmove_generic))))));
-# else
-		     : __memmove_generic)))));
-# endif
+extern __typeof (__redirect_memmove) __memmove_sve attribute_hidden;
+
+static inline __typeof (__redirect_memmove) *
+select_memmove_ifunc (void)
+{
+  INIT_ARCH ();
+
+  if (IS_NEOVERSE_N1 (midr) || IS_NEOVERSE_N2 (midr))
+    return __memmove_simd;
+
+  if (sve && HAVE_AARCH64_SVE_ASM)
+    {
+      if (IS_A64FX (midr))
+	return __memmove_a64fx;
+      return __memmove_sve;
+    }
+
+  if (IS_THUNDERX (midr))
+    return __memmove_thunderx;
+
+  if (IS_THUNDERX2 (midr) || IS_THUNDERX2PA (midr))
+    return __memmove_thunderx2;
+
+  if (IS_FALKOR (midr) || IS_PHECDA (midr))
+    return __memmove_falkor;
+
+  return __memmove_generic;
+}
+
+libc_ifunc (__libc_memmove, select_memmove_ifunc ());
+
 # undef memmove
 strong_alias (__libc_memmove, memmove);
 #endif