about summary refs log tree commit diff
path: root/sysdeps
diff options
context:
space:
mode:
Diffstat (limited to 'sysdeps')
-rw-r--r--sysdeps/x86_64/multiarch/Makefile5
-rw-r--r--sysdeps/x86_64/multiarch/ifunc-impl-list.c24
-rw-r--r--sysdeps/x86_64/multiarch/ifunc-memmove.h80
-rw-r--r--sysdeps/x86_64/multiarch/memcpy.S75
-rw-r--r--sysdeps/x86_64/multiarch/memcpy.c39
-rw-r--r--sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S21
-rw-r--r--sysdeps/x86_64/multiarch/memcpy_chk.S72
-rw-r--r--sysdeps/x86_64/multiarch/memcpy_chk.c31
-rw-r--r--sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S35
-rw-r--r--sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S11
-rw-r--r--sysdeps/x86_64/multiarch/memmove.S101
-rw-r--r--sysdeps/x86_64/multiarch/memmove.c38
-rw-r--r--sysdeps/x86_64/multiarch/memmove_chk-nonshared.S21
-rw-r--r--sysdeps/x86_64/multiarch/memmove_chk.S71
-rw-r--r--sysdeps/x86_64/multiarch/memmove_chk.c31
-rw-r--r--sysdeps/x86_64/multiarch/mempcpy.S73
-rw-r--r--sysdeps/x86_64/multiarch/mempcpy.c40
-rw-r--r--sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S21
-rw-r--r--sysdeps/x86_64/multiarch/mempcpy_chk.S72
-rw-r--r--sysdeps/x86_64/multiarch/mempcpy_chk.c31
20 files changed, 418 insertions, 474 deletions
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index 202a19c492..6e70993a45 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -26,6 +26,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \
 		   strchr-sse2-no-bsf memcmp-ssse3 strstr-sse2-unaligned \
 		   strcspn-c strpbrk-c strspn-c varshift \
 		   memset-avx512-no-vzeroupper \
+		   memmove-sse2-unaligned-erms \
 		   memmove-avx-unaligned-erms \
 		   memmove-avx512-unaligned-erms \
 		   memset-avx2-unaligned-erms \
@@ -48,5 +49,7 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \
 endif
 
 ifeq ($(subdir),debug)
-sysdep_routines += wmemset_chk-nonshared
+sysdep_routines += memcpy_chk-nonshared mempcpy_chk-nonshared \
+		   memmove_chk-nonshared \
+		   wmemset_chk-nonshared
 endif
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index 1385b1f3b0..3462e67cc7 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -57,7 +57,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 			      __memcmp_ssse3)
 	      IFUNC_IMPL_ADD (array, i, memcmp, 1, __memcmp_sse2))
 
-  /* Support sysdeps/x86_64/multiarch/memmove_chk.S.  */
+  /* Support sysdeps/x86_64/multiarch/memmove_chk.c.  */
   IFUNC_IMPL (i, name, __memmove_chk,
 	      IFUNC_IMPL_ADD (array, i, __memmove_chk,
 			      HAS_ARCH_FEATURE (AVX512F_Usable),
@@ -83,9 +83,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 	      IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
 			      __memmove_chk_sse2_unaligned)
 	      IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
-			      __memmove_chk_sse2_unaligned_erms))
+			      __memmove_chk_sse2_unaligned_erms)
+	      IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
+			      __memmove_chk_erms))
 
-  /* Support sysdeps/x86_64/multiarch/memmove.S.  */
+  /* Support sysdeps/x86_64/multiarch/memmove.c.  */
   IFUNC_IMPL (i, name, memmove,
 	      IFUNC_IMPL_ADD (array, i, memmove,
 			      HAS_ARCH_FEATURE (AVX_Usable),
@@ -410,7 +412,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 			      __wmemset_avx512_unaligned))
 
 #ifdef SHARED
-  /* Support sysdeps/x86_64/multiarch/memcpy_chk.S.  */
+  /* Support sysdeps/x86_64/multiarch/memcpy_chk.c.  */
   IFUNC_IMPL (i, name, __memcpy_chk,
 	      IFUNC_IMPL_ADD (array, i, __memcpy_chk,
 			      HAS_ARCH_FEATURE (AVX512F_Usable),
@@ -436,9 +438,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 	      IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
 			      __memcpy_chk_sse2_unaligned)
 	      IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
-			      __memcpy_chk_sse2_unaligned_erms))
+			      __memcpy_chk_sse2_unaligned_erms)
+	      IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
+			      __memcpy_chk_erms))
 
-  /* Support sysdeps/x86_64/multiarch/memcpy.S.  */
+  /* Support sysdeps/x86_64/multiarch/memcpy.c.  */
   IFUNC_IMPL (i, name, memcpy,
 	      IFUNC_IMPL_ADD (array, i, memcpy,
 			      HAS_ARCH_FEATURE (AVX_Usable),
@@ -464,7 +468,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 			      __memcpy_sse2_unaligned_erms)
 	      IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_erms))
 
-  /* Support sysdeps/x86_64/multiarch/mempcpy_chk.S.  */
+  /* Support sysdeps/x86_64/multiarch/mempcpy_chk.c.  */
   IFUNC_IMPL (i, name, __mempcpy_chk,
 	      IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
 			      HAS_ARCH_FEATURE (AVX512F_Usable),
@@ -490,9 +494,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 	      IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
 			      __mempcpy_chk_sse2_unaligned)
 	      IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
-			      __mempcpy_chk_sse2_unaligned_erms))
+			      __mempcpy_chk_sse2_unaligned_erms)
+	      IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
+			      __mempcpy_chk_erms))
 
-  /* Support sysdeps/x86_64/multiarch/mempcpy.S.  */
+  /* Support sysdeps/x86_64/multiarch/mempcpy.c.  */
   IFUNC_IMPL (i, name, mempcpy,
 	      IFUNC_IMPL_ADD (array, i, mempcpy,
 			      HAS_ARCH_FEATURE (AVX512F_Usable),
diff --git a/sysdeps/x86_64/multiarch/ifunc-memmove.h b/sysdeps/x86_64/multiarch/ifunc-memmove.h
new file mode 100644
index 0000000000..5951e156fa
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/ifunc-memmove.h
@@ -0,0 +1,80 @@
+/* Common definition for memcpy, mempcpy and memmove implementation.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <init-arch.h>
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (erms) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned)
+  attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned_erms)
+  attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3_back) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms)
+  attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned)
+  attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms)
+  attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_no_vzeroupper)
+  attribute_hidden;
+
+static inline void *
+IFUNC_SELECTOR (void)
+{
+  const struct cpu_features* cpu_features = __get_cpu_features ();
+
+  if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_ERMS))
+    return OPTIMIZE (erms);
+
+  if (CPU_FEATURES_ARCH_P (cpu_features, AVX512F_Usable)
+      && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
+    {
+      if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
+	return OPTIMIZE (avx512_no_vzeroupper);
+
+      if (CPU_FEATURES_CPU_P (cpu_features, ERMS))
+	return OPTIMIZE (avx512_unaligned_erms);
+
+      return OPTIMIZE (avx512_unaligned);
+    }
+
+  if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
+    {
+      if (CPU_FEATURES_CPU_P (cpu_features, ERMS))
+	return OPTIMIZE (avx_unaligned_erms);
+
+      return OPTIMIZE (avx_unaligned);
+    }
+
+  if (!CPU_FEATURES_CPU_P (cpu_features, SSSE3)
+      || CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Copy))
+    {
+      if (CPU_FEATURES_CPU_P (cpu_features, ERMS))
+	return OPTIMIZE (sse2_unaligned_erms);
+
+      return OPTIMIZE (sse2_unaligned);
+    }
+
+  if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Copy_Backward))
+    return OPTIMIZE (ssse3_back);
+
+  return OPTIMIZE (ssse3);
+}
diff --git a/sysdeps/x86_64/multiarch/memcpy.S b/sysdeps/x86_64/multiarch/memcpy.S
deleted file mode 100644
index af2770397c..0000000000
--- a/sysdeps/x86_64/multiarch/memcpy.S
+++ /dev/null
@@ -1,75 +0,0 @@
-/* Multiple versions of memcpy
-   All versions must be listed in ifunc-impl-list.c.
-   Copyright (C) 2010-2017 Free Software Foundation, Inc.
-   Contributed by Intel Corporation.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <sysdep.h>
-#include <init-arch.h>
-
-/* Define multiple versions only for the definition in lib and for
-   DSO.  In static binaries we need memcpy before the initialization
-   happened.  */
-#if defined SHARED && IS_IN (libc)
-	.text
-ENTRY(__new_memcpy)
-	.type	__new_memcpy, @gnu_indirect_function
-	LOAD_RTLD_GLOBAL_RO_RDX
-	lea	__memcpy_erms(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Prefer_ERMS)
-	jnz	2f
-	HAS_ARCH_FEATURE (Prefer_No_AVX512)
-	jnz	1f
-	HAS_ARCH_FEATURE (AVX512F_Usable)
-	jz	1f
-	lea	__memcpy_avx512_no_vzeroupper(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
-	jnz	2f
-	lea	__memcpy_avx512_unaligned_erms(%rip), %RAX_LP
-	HAS_CPU_FEATURE (ERMS)
-	jnz	2f
-	lea	__memcpy_avx512_unaligned(%rip), %RAX_LP
-	ret
-1:	lea	__memcpy_avx_unaligned(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
-	jz	L(Fast_Unaligned_Load)
-	HAS_CPU_FEATURE (ERMS)
-	jz	2f
-	lea	__memcpy_avx_unaligned_erms(%rip), %RAX_LP
-	ret
-L(Fast_Unaligned_Load):
-	lea	__memcpy_sse2_unaligned(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
-	jz	L(SSSE3)
-	HAS_CPU_FEATURE (ERMS)
-	jz	2f
-	lea	__memcpy_sse2_unaligned_erms(%rip), %RAX_LP
-	ret
-L(SSSE3):
-	HAS_CPU_FEATURE (SSSE3)
-	jz	2f
-	lea    __memcpy_ssse3_back(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Fast_Copy_Backward)
-	jnz	2f
-	lea	__memcpy_ssse3(%rip), %RAX_LP
-2:	ret
-END(__new_memcpy)
-
-# undef memcpy
-# include <shlib-compat.h>
-versioned_symbol (libc, __new_memcpy, memcpy, GLIBC_2_14);
-#endif
diff --git a/sysdeps/x86_64/multiarch/memcpy.c b/sysdeps/x86_64/multiarch/memcpy.c
new file mode 100644
index 0000000000..6a2d3539e2
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memcpy.c
@@ -0,0 +1,39 @@
+/* Multiple versions of memcpy.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in lib and for
+   DSO.  In static binaries we need memcpy before the initialization
+   happened.  */
+#if defined SHARED && IS_IN (libc)
+# define memcpy __redirect_memcpy
+# include <string.h>
+# undef memcpy
+
+# define SYMBOL_NAME memcpy
+# include "ifunc-memmove.h"
+
+libc_ifunc_redirected (__redirect_memcpy, __new_memcpy,
+		       IFUNC_SELECTOR ());
+
+__hidden_ver1 (__new_memcpy, __GI_memcpy, __redirect_memcpy)
+  __attribute__ ((visibility ("hidden")));
+
+# include <shlib-compat.h>
+versioned_symbol (libc, __new_memcpy, memcpy, GLIBC_2_14);
+#endif
diff --git a/sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S b/sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S
new file mode 100644
index 0000000000..41eb0e81f6
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S
@@ -0,0 +1,21 @@
+/* Non-shared version of memcpy_chk for x86-64.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#if IS_IN (libc) && !defined SHARED
+# include <sysdeps/x86_64/memcpy_chk.S>
+#endif
diff --git a/sysdeps/x86_64/multiarch/memcpy_chk.S b/sysdeps/x86_64/multiarch/memcpy_chk.S
deleted file mode 100644
index 8737fb9755..0000000000
--- a/sysdeps/x86_64/multiarch/memcpy_chk.S
+++ /dev/null
@@ -1,72 +0,0 @@
-/* Multiple versions of __memcpy_chk
-   All versions must be listed in ifunc-impl-list.c.
-   Copyright (C) 2010-2017 Free Software Foundation, Inc.
-   Contributed by Intel Corporation.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <sysdep.h>
-#include <init-arch.h>
-
-/* Define multiple versions only for the definition in lib and for
-   DSO.  There are no multiarch memcpy functions for static binaries.
- */
-#if IS_IN (libc)
-# ifdef SHARED
-	.text
-ENTRY(__memcpy_chk)
-	.type	__memcpy_chk, @gnu_indirect_function
-	LOAD_RTLD_GLOBAL_RO_RDX
-	HAS_ARCH_FEATURE (Prefer_No_AVX512)
-	jnz	1f
-	HAS_ARCH_FEATURE (AVX512F_Usable)
-	jz	1f
-	lea	__memcpy_chk_avx512_no_vzeroupper(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
-	jnz	2f
-	lea	__memcpy_chk_avx512_unaligned_erms(%rip), %RAX_LP
-	HAS_CPU_FEATURE (ERMS)
-	jnz	2f
-	lea	__memcpy_chk_avx512_unaligned(%rip), %RAX_LP
-	ret
-1:	lea	__memcpy_chk_avx_unaligned(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
-	jz	L(Fast_Unaligned_Load)
-	HAS_CPU_FEATURE (ERMS)
-	jz	2f
-	lea	__memcpy_chk_avx_unaligned_erms(%rip), %RAX_LP
-	ret
-L(Fast_Unaligned_Load):
-	lea	__memcpy_chk_sse2_unaligned(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
-	jz	L(SSSE3)
-	HAS_CPU_FEATURE (ERMS)
-	jz	2f
-	lea	__memcpy_chk_sse2_unaligned_erms(%rip), %RAX_LP
-	ret
-L(SSSE3):
-	HAS_CPU_FEATURE (SSSE3)
-	jz	2f
-	lea    __memcpy_chk_ssse3_back(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Fast_Copy_Backward)
-	jnz	2f
-	lea	__memcpy_chk_ssse3(%rip), %RAX_LP
-2:	ret
-END(__memcpy_chk)
-# else
-#  include "../memcpy_chk.S"
-# endif
-#endif
diff --git a/sysdeps/x86_64/multiarch/memcpy_chk.c b/sysdeps/x86_64/multiarch/memcpy_chk.c
new file mode 100644
index 0000000000..6347c21e3c
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memcpy_chk.c
@@ -0,0 +1,31 @@
+/* Multiple versions of __memcpy_chk
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in libc.so. */
+#if IS_IN (libc) && defined SHARED
+# define __memcpy_chk __redirect_memcpy_chk
+# include <string.h>
+# undef __memcpy_chk
+
+# define SYMBOL_NAME memcpy_chk
+# include "ifunc-memmove.h"
+
+libc_ifunc_redirected (__redirect_memcpy_chk, __memcpy_chk,
+		       IFUNC_SELECTOR ());
+#endif
diff --git a/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S
new file mode 100644
index 0000000000..743064b63d
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S
@@ -0,0 +1,35 @@
+/* memmove with SSE2.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#if IS_IN (libc)
+# define MEMMOVE_SYMBOL(p,s)	p##_sse2_##s
+#endif
+
+#if !defined SHARED || !IS_IN (libc)
+weak_alias (__mempcpy, mempcpy)
+#endif
+
+#include <sysdeps/x86_64/memmove.S>
+
+#if defined SHARED && IS_IN (libc)
+# include <shlib-compat.h>
+# if SHLIB_COMPAT (libc, GLIBC_2_2_5, GLIBC_2_14)
+/* Use __memmove_sse2_unaligned to support overlapping addresses.  */
+compat_symbol (libc, __memmove_sse2_unaligned, memcpy, GLIBC_2_2_5);
+# endif
+#endif
diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
index dee3ec529c..d694e8b2be 100644
--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
@@ -152,6 +152,11 @@ END (MEMMOVE_SYMBOL (__memmove, unaligned))
 
 # if VEC_SIZE == 16
 #  if defined SHARED
+ENTRY (__mempcpy_chk_erms)
+	cmpq	%rdx, %rcx
+	jb	HIDDEN_JUMPTARGET (__chk_fail)
+END (__mempcpy_chk_erms)
+
 /* Only used to measure performance of REP MOVSB.  */
 ENTRY (__mempcpy_erms)
 	movq	%rdi, %rax
@@ -160,6 +165,11 @@ ENTRY (__mempcpy_erms)
 END (__mempcpy_erms)
 #  endif
 
+ENTRY (__memmove_chk_erms)
+	cmpq	%rdx, %rcx
+	jb	HIDDEN_JUMPTARGET (__chk_fail)
+END (__memmove_chk_erms)
+
 ENTRY (__memmove_erms)
 	movq	%rdi, %rax
 L(start_movsb):
@@ -185,6 +195,7 @@ L(movsb_backward):
 END (__memmove_erms)
 #  if defined SHARED
 strong_alias (__memmove_erms, __memcpy_erms)
+strong_alias (__memmove_chk_erms, __memcpy_chk_erms)
 #  endif
 # endif
 
diff --git a/sysdeps/x86_64/multiarch/memmove.S b/sysdeps/x86_64/multiarch/memmove.S
deleted file mode 100644
index 8c534e83e0..0000000000
--- a/sysdeps/x86_64/multiarch/memmove.S
+++ /dev/null
@@ -1,101 +0,0 @@
-/* Multiple versions of memmove
-   All versions must be listed in ifunc-impl-list.c.
-   Copyright (C) 2016-2017 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <sysdep.h>
-#include <init-arch.h>
-
-/* Define multiple versions only for the definition in lib and for
-   DSO.  */
-#if IS_IN (libc)
-	.text
-ENTRY(__libc_memmove)
-	.type	__libc_memmove, @gnu_indirect_function
-	LOAD_RTLD_GLOBAL_RO_RDX
-	lea	__memmove_erms(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Prefer_ERMS)
-	jnz	2f
-	HAS_ARCH_FEATURE (Prefer_No_AVX512)
-	jnz	1f
-	HAS_ARCH_FEATURE (AVX512F_Usable)
-	jz	1f
-	lea	__memmove_avx512_no_vzeroupper(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
-	jnz	2f
-	lea	__memmove_avx512_unaligned_erms(%rip), %RAX_LP
-	HAS_CPU_FEATURE (ERMS)
-	jnz	2f
-	lea	__memmove_avx512_unaligned(%rip), %RAX_LP
-	ret
-1:	lea	__memmove_avx_unaligned(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
-	jz	L(Fast_Unaligned_Load)
-	HAS_CPU_FEATURE (ERMS)
-	jz	2f
-	lea	__memmove_avx_unaligned_erms(%rip), %RAX_LP
-	ret
-L(Fast_Unaligned_Load):
-	lea	__memmove_sse2_unaligned(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
-	jz	L(SSSE3)
-	HAS_CPU_FEATURE (ERMS)
-	jz	2f
-	lea	__memmove_sse2_unaligned_erms(%rip), %RAX_LP
-	ret
-L(SSSE3):
-	HAS_CPU_FEATURE (SSSE3)
-	jz	2f
-	lea    __memmove_ssse3_back(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Fast_Copy_Backward)
-	jnz	2f
-	lea	__memmove_ssse3(%rip), %RAX_LP
-2:	ret
-END(__libc_memmove)
-#endif
-
-#if IS_IN (libc)
-# define MEMMOVE_SYMBOL(p,s)	p##_sse2_##s
-
-# ifdef SHARED
-libc_hidden_ver (__memmove_sse2_unaligned, memmove)
-libc_hidden_ver (__memcpy_sse2_unaligned, memcpy)
-libc_hidden_ver (__mempcpy_sse2_unaligned, mempcpy)
-libc_hidden_ver (__mempcpy_sse2_unaligned, __mempcpy)
-
-#  undef libc_hidden_builtin_def
-/* It doesn't make sense to send libc-internal memmove calls through a PLT.
-   The speedup we get from using SSE2 instructions is likely eaten away
-   by the indirect call in the PLT.  */
-#  define libc_hidden_builtin_def
-# endif
-strong_alias (__libc_memmove, memmove)
-#endif
-
-#if !defined SHARED || !IS_IN (libc)
-weak_alias (__mempcpy, mempcpy)
-#endif
-
-#include "../memmove.S"
-
-#if defined SHARED && IS_IN (libc)
-# include <shlib-compat.h>
-# if SHLIB_COMPAT (libc, GLIBC_2_2_5, GLIBC_2_14)
-/* Use __memmove_sse2_unaligned to support overlapping addresses.  */
-compat_symbol (libc, __memmove_sse2_unaligned, memcpy, GLIBC_2_2_5);
-# endif
-#endif
diff --git a/sysdeps/x86_64/multiarch/memmove.c b/sysdeps/x86_64/multiarch/memmove.c
new file mode 100644
index 0000000000..c615808cc1
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memmove.c
@@ -0,0 +1,38 @@
+/* Multiple versions of memmmove.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2016-2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in lib and for
+   DSO.  */
+#if IS_IN (libc)
+# define memmove __redirect_memmove
+# include <string.h>
+# undef memmove
+
+# define SYMBOL_NAME memmove
+# include "ifunc-memmove.h"
+
+libc_ifunc_redirected (__redirect_memmove, __libc_memmove,
+		       IFUNC_SELECTOR ());
+
+strong_alias (__libc_memmove, memmove);
+# ifdef SHARED
+__hidden_ver1 (__libc_memmove, __GI_memmove, __redirect_memmove)
+  __attribute__ ((visibility ("hidden")));
+# endif
+#endif
diff --git a/sysdeps/x86_64/multiarch/memmove_chk-nonshared.S b/sysdeps/x86_64/multiarch/memmove_chk-nonshared.S
new file mode 100644
index 0000000000..704a20c408
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memmove_chk-nonshared.S
@@ -0,0 +1,21 @@
+/* Non-shared version of memmove_chk for x86-64.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#if IS_IN (libc) && !defined SHARED
+# include <sysdeps/x86_64/memmove_chk.S>
+#endif
diff --git a/sysdeps/x86_64/multiarch/memmove_chk.S b/sysdeps/x86_64/multiarch/memmove_chk.S
deleted file mode 100644
index 7870dd0247..0000000000
--- a/sysdeps/x86_64/multiarch/memmove_chk.S
+++ /dev/null
@@ -1,71 +0,0 @@
-/* Multiple versions of __memmove_chk
-   All versions must be listed in ifunc-impl-list.c.
-   Copyright (C) 2016-2017 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <sysdep.h>
-#include <init-arch.h>
-
-/* Define multiple versions only for the definition in lib and for
-   DSO.  There are no multiarch memmove functions for static binaries.
- */
-#if IS_IN (libc)
-# ifdef SHARED
-	.text
-ENTRY(__memmove_chk)
-	.type	__memmove_chk, @gnu_indirect_function
-	LOAD_RTLD_GLOBAL_RO_RDX
-	HAS_ARCH_FEATURE (Prefer_No_AVX512)
-	jnz	1f
-	HAS_ARCH_FEATURE (AVX512F_Usable)
-	jz	1f
-	lea	__memmove_chk_avx512_no_vzeroupper(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
-	jnz	2f
-	lea	__memmove_chk_avx512_unaligned_erms(%rip), %RAX_LP
-	HAS_CPU_FEATURE (ERMS)
-	jnz	2f
-	lea	__memmove_chk_avx512_unaligned(%rip), %RAX_LP
-	ret
-1:	lea	__memmove_chk_avx_unaligned(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
-	jz	L(Fast_Unaligned_Load)
-	HAS_CPU_FEATURE (ERMS)
-	jz	2f
-	lea	__memmove_chk_avx_unaligned_erms(%rip), %RAX_LP
-	ret
-L(Fast_Unaligned_Load):
-	lea	__memmove_chk_sse2_unaligned(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
-	jz	L(SSSE3)
-	HAS_CPU_FEATURE (ERMS)
-	jz	2f
-	lea	__memmove_chk_sse2_unaligned_erms(%rip), %RAX_LP
-	ret
-L(SSSE3):
-	HAS_CPU_FEATURE (SSSE3)
-	jz	2f
-	lea    __memmove_chk_ssse3_back(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Fast_Copy_Backward)
-	jnz	2f
-	lea	__memmove_chk_ssse3(%rip), %RAX_LP
-2:	ret
-END(__memmove_chk)
-# else
-#  include "../memmove_chk.S"
-# endif
-#endif
diff --git a/sysdeps/x86_64/multiarch/memmove_chk.c b/sysdeps/x86_64/multiarch/memmove_chk.c
new file mode 100644
index 0000000000..44339d64ec
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memmove_chk.c
@@ -0,0 +1,31 @@
+/* Multiple versions of __memmove_chk
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in libc.so. */
+#if IS_IN (libc) && defined SHARED
+# define __memmove_chk __redirect_memmove_chk
+# include <string.h>
+# undef __memmove_chk
+
+# define SYMBOL_NAME memmove_chk
+# include "ifunc-memmove.h"
+
+libc_ifunc_redirected (__redirect_memmove_chk, __memmove_chk,
+		       IFUNC_SELECTOR ());
+#endif
diff --git a/sysdeps/x86_64/multiarch/mempcpy.S b/sysdeps/x86_64/multiarch/mempcpy.S
deleted file mode 100644
index b8b2b28094..0000000000
--- a/sysdeps/x86_64/multiarch/mempcpy.S
+++ /dev/null
@@ -1,73 +0,0 @@
-/* Multiple versions of mempcpy
-   All versions must be listed in ifunc-impl-list.c.
-   Copyright (C) 2010-2017 Free Software Foundation, Inc.
-   Contributed by Intel Corporation.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <sysdep.h>
-#include <init-arch.h>
-
-/* Define multiple versions only for the definition in lib and for
-   DSO.  In static binaries we need mempcpy before the initialization
-   happened.  */
-#if defined SHARED && IS_IN (libc)
-	.text
-ENTRY(__mempcpy)
-	.type	__mempcpy, @gnu_indirect_function
-	LOAD_RTLD_GLOBAL_RO_RDX
-	lea	__mempcpy_erms(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Prefer_ERMS)
-	jnz	2f
-	HAS_ARCH_FEATURE (Prefer_No_AVX512)
-	jnz	1f
-	HAS_ARCH_FEATURE (AVX512F_Usable)
-	jz	1f
-	lea	__mempcpy_avx512_no_vzeroupper(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
-	jnz	2f
-	lea	__mempcpy_avx512_unaligned_erms(%rip), %RAX_LP
-	HAS_CPU_FEATURE (ERMS)
-	jnz	2f
-	lea	__mempcpy_avx512_unaligned(%rip), %RAX_LP
-	ret
-1:	lea	__mempcpy_avx_unaligned(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
-	jz	L(Fast_Unaligned_Load)
-	HAS_CPU_FEATURE (ERMS)
-	jz	2f
-	lea	__mempcpy_avx_unaligned_erms(%rip), %RAX_LP
-	ret
-L(Fast_Unaligned_Load):
-	lea	__mempcpy_sse2_unaligned(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
-	jz	L(SSSE3)
-	HAS_CPU_FEATURE (ERMS)
-	jz	2f
-	lea	__mempcpy_sse2_unaligned_erms(%rip), %RAX_LP
-	ret
-L(SSSE3):
-	HAS_CPU_FEATURE (SSSE3)
-	jz	2f
-	lea    __mempcpy_ssse3_back(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Fast_Copy_Backward)
-	jnz	2f
-	lea	__mempcpy_ssse3(%rip), %RAX_LP
-2:	ret
-END(__mempcpy)
-
-weak_alias (__mempcpy, mempcpy)
-#endif
diff --git a/sysdeps/x86_64/multiarch/mempcpy.c b/sysdeps/x86_64/multiarch/mempcpy.c
new file mode 100644
index 0000000000..f6e9b8acc0
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/mempcpy.c
@@ -0,0 +1,40 @@
+/* Multiple versions of mempcpy.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in lib and for
+   DSO.  In static binaries we need mempcpy before the initialization
+   happened.  */
+#if defined SHARED && IS_IN (libc)
+# define mempcpy __redirect_mempcpy
+# define __mempcpy __redirect___mempcpy
+# include <string.h>
+# undef mempcpy
+# undef __mempcpy
+
+# define SYMBOL_NAME mempcpy
+# include "ifunc-memmove.h"
+
+libc_ifunc_redirected (__redirect_mempcpy, __mempcpy, IFUNC_SELECTOR ());
+
+weak_alias (__mempcpy, mempcpy)
+__hidden_ver1 (__mempcpy, __GI___mempcpy, __redirect___mempcpy)
+  __attribute__ ((visibility ("hidden")));
+__hidden_ver1 (mempcpy, __GI_mempcpy, __redirect_mempcpy)
+  __attribute__ ((visibility ("hidden")));
+#endif
diff --git a/sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S b/sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S
new file mode 100644
index 0000000000..01b7df7f8c
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S
@@ -0,0 +1,21 @@
+/* Non-shared version of mempcpy_chk for x86-64.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#if IS_IN (libc) && !defined SHARED
+# include <sysdeps/x86_64/mempcpy_chk.S>
+#endif
diff --git a/sysdeps/x86_64/multiarch/mempcpy_chk.S b/sysdeps/x86_64/multiarch/mempcpy_chk.S
deleted file mode 100644
index 072b22c49f..0000000000
--- a/sysdeps/x86_64/multiarch/mempcpy_chk.S
+++ /dev/null
@@ -1,72 +0,0 @@
-/* Multiple versions of __mempcpy_chk
-   All versions must be listed in ifunc-impl-list.c.
-   Copyright (C) 2010-2017 Free Software Foundation, Inc.
-   Contributed by Intel Corporation.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <sysdep.h>
-#include <init-arch.h>
-
-/* Define multiple versions only for the definition in lib and for
-   DSO.  There are no multiarch mempcpy functions for static binaries.
- */
-#if IS_IN (libc)
-# ifdef SHARED
-	.text
-ENTRY(__mempcpy_chk)
-	.type	__mempcpy_chk, @gnu_indirect_function
-	LOAD_RTLD_GLOBAL_RO_RDX
-	HAS_ARCH_FEATURE (Prefer_No_AVX512)
-	jnz	1f
-	HAS_ARCH_FEATURE (AVX512F_Usable)
-	jz	1f
-	lea	__mempcpy_chk_avx512_no_vzeroupper(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
-	jnz	2f
-	lea	__mempcpy_chk_avx512_unaligned_erms(%rip), %RAX_LP
-	HAS_CPU_FEATURE (ERMS)
-	jnz	2f
-	lea	__mempcpy_chk_avx512_unaligned(%rip), %RAX_LP
-	ret
-1:	lea	__mempcpy_chk_avx_unaligned(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
-	jz	L(Fast_Unaligned_Load)
-	HAS_CPU_FEATURE (ERMS)
-	jz	2f
-	lea	__mempcpy_chk_avx_unaligned_erms(%rip), %RAX_LP
-	ret
-L(Fast_Unaligned_Load):
-	lea	__mempcpy_chk_sse2_unaligned(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
-	jz	L(SSSE3)
-	HAS_CPU_FEATURE (ERMS)
-	jz	2f
-	lea	__mempcpy_chk_sse2_unaligned_erms(%rip), %RAX_LP
-	ret
-L(SSSE3):
-	HAS_CPU_FEATURE (SSSE3)
-	jz	2f
-	lea    __mempcpy_chk_ssse3_back(%rip), %RAX_LP
-	HAS_ARCH_FEATURE (Fast_Copy_Backward)
-	jnz	2f
-	lea	__mempcpy_chk_ssse3(%rip), %RAX_LP
-2:	ret
-END(__mempcpy_chk)
-# else
-#  include "../mempcpy_chk.S"
-# endif
-#endif
diff --git a/sysdeps/x86_64/multiarch/mempcpy_chk.c b/sysdeps/x86_64/multiarch/mempcpy_chk.c
new file mode 100644
index 0000000000..c73273ece3
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/mempcpy_chk.c
@@ -0,0 +1,31 @@
+/* Multiple versions of __mempcpy_chk
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in libc.so. */
+#if IS_IN (libc) && defined SHARED
+# define __mempcpy_chk __redirect_mempcpy_chk
+# include <string.h>
+# undef __mempcpy_chk
+
+# define SYMBOL_NAME mempcpy_chk
+# include "ifunc-memmove.h"
+
+libc_ifunc_redirected (__redirect_mempcpy_chk, __mempcpy_chk,
+		       IFUNC_SELECTOR ());
+#endif