Add i386 memcpy family multiarch functions

34 files changed, 607 insertions, 109 deletions

diff --git a/sysdeps/i386/i686/multiarch/memset_chk.S b/sysdeps/i386/i686/multiarch/memset_chk.S
deleted file mode 100644
index 7be45e710e..0000000000
--- a/sysdeps/i386/i686/multiarch/memset_chk.S
+++ /dev/null
@@ -1,82 +0,0 @@
-/* Multiple versions of __memset_chk
-   All versions must be listed in ifunc-impl-list.c.
-   Copyright (C) 2010-2015 Free Software Foundation, Inc.
-   Contributed by Intel Corporation.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <sysdep.h>
-#include <init-arch.h>
-
-/* Define multiple versions only for the definition in lib.  */
-#if IS_IN (libc)
-	.text
-ENTRY(__memset_chk)
-	.type	__memset_chk, @gnu_indirect_function
-	LOAD_GOT_AND_RTLD_GLOBAL_RO
-	LOAD_FUNC_GOT_EAX (__memset_chk_ia32)
-	HAS_CPU_FEATURE (SSE2)
-	jz	2f
-	LOAD_FUNC_GOT_EAX (__memset_chk_sse2)
-	HAS_CPU_FEATURE (Fast_Rep_String)
-	jz	2f
-	LOAD_FUNC_GOT_EAX (__memset_chk_sse2_rep)
-2:	ret
-END(__memset_chk)
-
-# ifdef SHARED
-strong_alias (__memset_chk, __memset_zero_constant_len_parameter)
-	.section .gnu.warning.__memset_zero_constant_len_parameter
-	.string "memset used with constant zero length parameter; this could be due to transposed parameters"
-# else
-	.text
-	.type __memset_chk_sse2, @function
-	.p2align 4;
-__memset_chk_sse2:
-	cfi_startproc
-	CALL_MCOUNT
-	movl	12(%esp), %eax
-	cmpl	%eax, 16(%esp)
-	jb	__chk_fail
-	jmp	__memset_sse2
-	cfi_endproc
-	.size __memset_chk_sse2, .-__memset_chk_sse2
-
-	.type __memset_chk_sse2_rep, @function
-	.p2align 4;
-__memset_chk_sse2_rep:
-	cfi_startproc
-	CALL_MCOUNT
-	movl	12(%esp), %eax
-	cmpl	%eax, 16(%esp)
-	jb	__chk_fail
-	jmp	__memset_sse2_rep
-	cfi_endproc
-	.size __memset_chk_sse2_rep, .-__memset_chk_sse2_rep
-
-	.type __memset_chk_ia32, @function
-	.p2align 4;
-__memset_chk_ia32:
-	cfi_startproc
-	CALL_MCOUNT
-	movl	12(%esp), %eax
-	cmpl	%eax, 16(%esp)
-	jb	__chk_fail
-	jmp	__memset_ia32
-	cfi_endproc
-	.size __memset_chk_ia32, .-__memset_chk_ia32
-# endif
-#endif
diff --git a/sysdeps/i386/multiarch/Makefile b/sysdeps/i386/multiarch/Makefile
index cb96d46e98..63a61d8efe 100644
--- a/sysdeps/i386/multiarch/Makefile
+++ b/sysdeps/i386/multiarch/Makefile
@@ -5,7 +5,16 @@ endif
 
 ifeq ($(subdir),string)
 gen-as-const-headers += locale-defines.sym
-sysdep_routines += bzero-i386 bzero-i586 bzero-i686 \
+sysdep_routines += bcopy-i386 bcopy-i686 bcopy-sse2-unaligned \
+		   bcopy-ssse3 bcopy-ssse3-rep \
+		   memcpy-i386 memcpy-i586 memcpy-i686 \
+		   memcpy-sse2-unaligned memcpy-ssse3 memcpy-ssse3-rep \
+		   memmove-i386 memmove-i686 memmove-sse2-unaligned \
+		   memmove-ssse3 memmove-ssse3-rep \
+		   static-memmove \
+		   mempcpy-i386 mempcpy-i586 mempcpy-i686 \
+		   mempcpy-sse2-unaligned mempcpy-ssse3 mempcpy-ssse3-rep \
+		   bzero-i386 bzero-i586 bzero-i686 \
 		   bzero-sse2 bzero-sse2-rep \
 		   memset-i386 memset-i586 memset-i686 \
 		   memset-sse2 memset-sse2-rep \
diff --git a/sysdeps/i386/multiarch/bcopy-i386.S b/sysdeps/i386/multiarch/bcopy-i386.S
new file mode 100644
index 0000000000..ad85776543
--- /dev/null
+++ b/sysdeps/i386/multiarch/bcopy-i386.S
@@ -0,0 +1,15 @@
+#include <init-arch.h>
+#if !SUPPORT_I686
+# define bcopy __bcopy_i386
+# undef libc_hidden_builtin_def
+# define libc_hidden_builtin_def(name)
+# undef weak_alias
+# define weak_alias(name, aliasname)
+# include <sysdeps/i386/bcopy.S>
+
+# ifdef SHARED
+	.globl __GI_bcopy
+	.hidden __GI_bcopy
+	__GI_bcopy = __bcopy_i386
+# endif
+#endif
diff --git a/sysdeps/i386/multiarch/bcopy-i686.S b/sysdeps/i386/multiarch/bcopy-i686.S
new file mode 100644
index 0000000000..81c706110e
--- /dev/null
+++ b/sysdeps/i386/multiarch/bcopy-i686.S
@@ -0,0 +1,13 @@
+#include <init-arch.h>
+#define bcopy __bcopy_i686
+#undef libc_hidden_builtin_def
+#define libc_hidden_builtin_def(name)
+#undef weak_alias
+#define weak_alias(name, aliasname)
+# include <sysdeps/i386/i686/bcopy.S>
+
+#if defined SHARED && SUPPORT_I686
+	.globl __GI_bcopy
+	.hidden __GI_bcopy
+	__GI_bcopy = __bcopy_i686
+#endif
diff --git a/sysdeps/i386/i686/multiarch/bcopy-sse2-unaligned.S b/sysdeps/i386/multiarch/bcopy-sse2-unaligned.S
index efef2a10dd..efef2a10dd 100644
--- a/sysdeps/i386/i686/multiarch/bcopy-sse2-unaligned.S
+++ b/sysdeps/i386/multiarch/bcopy-sse2-unaligned.S
diff --git a/sysdeps/i386/i686/multiarch/bcopy-ssse3-rep.S b/sysdeps/i386/multiarch/bcopy-ssse3-rep.S
index cbc8b420e8..cbc8b420e8 100644
--- a/sysdeps/i386/i686/multiarch/bcopy-ssse3-rep.S
+++ b/sysdeps/i386/multiarch/bcopy-ssse3-rep.S
diff --git a/sysdeps/i386/i686/multiarch/bcopy-ssse3.S b/sysdeps/i386/multiarch/bcopy-ssse3.S
index 36aac44b9c..36aac44b9c 100644
--- a/sysdeps/i386/i686/multiarch/bcopy-ssse3.S
+++ b/sysdeps/i386/multiarch/bcopy-ssse3.S
diff --git a/sysdeps/i386/multiarch/bcopy.c b/sysdeps/i386/multiarch/bcopy.c
new file mode 100644
index 0000000000..21ecf62d00
--- /dev/null
+++ b/sysdeps/i386/multiarch/bcopy.c
@@ -0,0 +1,64 @@
+/* Multiple versions of bcopy.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2015 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in lib.  */
+#if IS_IN (libc)
+/* Redefine bcopy so that the compiler won't complain about the type
+   mismatch with the IFUNC selector in strong_alias, below.  */
+# undef bcopy
+# define bcopy __redirect_bcopy
+# include <string.h>
+# undef bcopy
+
+# include <init-arch.h>
+
+extern __typeof (__redirect_bcopy) __bcopy_i386 attribute_hidden;
+extern __typeof (__redirect_bcopy) __bcopy_i686 attribute_hidden;
+extern __typeof (__redirect_bcopy) __bcopy_sse2_unaligned attribute_hidden;
+extern __typeof (__redirect_bcopy) __bcopy_ssse3 attribute_hidden;
+extern __typeof (__redirect_bcopy) __bcopy_ssse3_rep attribute_hidden;
+
+/* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle
+   ifunc symbol properly.  */
+extern __typeof (__redirect_bcopy) bcopy;
+extern void *bcopy_ifunc (void) __asm__ ("bcopy");
+
+void *
+bcopy_ifunc (void)
+{
+  if (HAS_CPU_FEATURE (SSE2))
+    {
+      if (HAS_ARCH_FEATURE (Fast_Unaligned_Load))
+	return __bcopy_sse2_unaligned;
+      else if (HAS_CPU_FEATURE (SSSE3))
+	{
+	  if (HAS_ARCH_FEATURE (Fast_Rep_String))
+	    return __bcopy_ssse3_rep;
+	  else
+	    return __bcopy_ssse3;
+	}
+    }
+
+  if (HAS_I686)
+    return __bcopy_i686;
+  else
+    return __bcopy_i386;
+}
+__asm__ (".type bcopy, %gnu_indirect_function");
+#endif
diff --git a/sysdeps/i386/multiarch/ifunc-impl-list.c b/sysdeps/i386/multiarch/ifunc-impl-list.c
index 87d2080f7c..18f17e4f18 100644
--- a/sysdeps/i386/multiarch/ifunc-impl-list.c
+++ b/sysdeps/i386/multiarch/ifunc-impl-list.c
@@ -36,7 +36,6 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 
   size_t i = 0;
 
-#if 0
   /* Support sysdeps/i386/i686/multiarch/bcopy.S.  */
   IFUNC_IMPL (i, name, bcopy,
 	      IFUNC_IMPL_ADD (array, i, bcopy, HAS_CPU_FEATURE (SSSE3),
@@ -45,8 +44,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 			      __bcopy_ssse3)
 	      IFUNC_IMPL_ADD (array, i, bcopy, HAS_CPU_FEATURE (SSE2),
 			      __bcopy_sse2_unaligned)
-	      IFUNC_IMPL_ADD (array, i, bcopy, 1, __bcopy_ia32))
+	      IFUNC_IMPL_ADD (array, i, bcopy, HAS_I686, __bcopy_i686)
+#if !SUPPORT_I686
+	      IFUNC_IMPL_ADD (array, i, bcopy, 1, __bcopy_i386)
 #endif
+	      )
 
   /* Support sysdeps/i386/i686/multiarch/bzero.S.  */
   IFUNC_IMPL (i, name, bzero,
@@ -79,6 +81,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 	      IFUNC_IMPL_ADD (array, i, memcmp, HAS_CPU_FEATURE (SSSE3),
 			      __memcmp_ssse3)
 	      IFUNC_IMPL_ADD (array, i, memcmp, 1, __memcmp_ia32))
+#endif
 
   /* Support sysdeps/i386/i686/multiarch/memmove_chk.S.  */
   IFUNC_IMPL (i, name, __memmove_chk,
@@ -91,8 +94,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 	      IFUNC_IMPL_ADD (array, i, __memmove_chk,
 			      HAS_CPU_FEATURE (SSE2),
 			      __memmove_chk_sse2_unaligned)
+	      IFUNC_IMPL_ADD (array, i, __memmove_chk, HAS_I686,
+			      __memmove_chk_i686)
+#if !SUPPORT_I686
 	      IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
-			      __memmove_chk_ia32))
+			      __memmove_chk_i386)
+#endif
+	      )
 
   /* Support sysdeps/i386/i686/multiarch/memmove.S.  */
   IFUNC_IMPL (i, name, memmove,
@@ -102,8 +110,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 			      __memmove_ssse3)
 	      IFUNC_IMPL_ADD (array, i, memmove, HAS_CPU_FEATURE (SSE2),
 			      __memmove_sse2_unaligned)
-	      IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_ia32))
+	      IFUNC_IMPL_ADD (array, i, memmove, HAS_I686, __memmove_i686)
+#if !SUPPORT_I686
+	      IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_i386)
+#endif
+	      )
 
+#if 0
   /* Support sysdeps/i386/i686/multiarch/memrchr.S.  */
   IFUNC_IMPL (i, name, memrchr,
 	      IFUNC_IMPL_ADD (array, i, memrchr, HAS_CPU_FEATURE (SSE2),
@@ -335,6 +348,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 	      IFUNC_IMPL_ADD (array, i, wmemcmp, HAS_CPU_FEATURE (SSSE3),
 			      __wmemcmp_ssse3)
 	      IFUNC_IMPL_ADD (array, i, wmemcmp, 1, __wmemcmp_ia32))
+#endif
 
 #ifdef SHARED
   /* Support sysdeps/i386/i686/multiarch/memcpy_chk.S.  */
@@ -348,8 +362,17 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 	      IFUNC_IMPL_ADD (array, i, __memcpy_chk,
 			      HAS_CPU_FEATURE (SSE2),
 			      __memcpy_chk_sse2_unaligned)
+	      IFUNC_IMPL_ADD (array, i, __memcpy_chk, HAS_I686,
+			      __memcpy_chk_i686)
+# if !SUPPORT_I686
+	      IFUNC_IMPL_ADD (array, i, __memcpy_chk, HAS_I586,
+			      __memcpy_chk_i586)
+#  if !SUPPORT_I586
 	      IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
-			      __memcpy_chk_ia32))
+			      __memcpy_chk_i386)
+#  endif
+# endif
+	      )
 
   /* Support sysdeps/i386/i686/multiarch/memcpy.S.  */
   IFUNC_IMPL (i, name, memcpy,
@@ -359,7 +382,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 			      __memcpy_ssse3)
 	      IFUNC_IMPL_ADD (array, i, memcpy, HAS_CPU_FEATURE (SSE2),
 			      __memcpy_sse2_unaligned)
-	      IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_ia32))
+	      IFUNC_IMPL_ADD (array, i, memcpy, HAS_I686, __memcpy_i686)
+# if !SUPPORT_I686
+	      IFUNC_IMPL_ADD (array, i, memcpy, HAS_I586, __memcpy_i586)
+#  if !SUPPORT_I586
+	      IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_i386)
+#  endif
+# endif
+	      )
 
   /* Support sysdeps/i386/i686/multiarch/mempcpy_chk.S.  */
   IFUNC_IMPL (i, name, __mempcpy_chk,
@@ -372,8 +402,17 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 	      IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
 			      HAS_CPU_FEATURE (SSE2),
 			      __mempcpy_chk_sse2_unaligned)
+	      IFUNC_IMPL_ADD (array, i, __mempcpy_chk, HAS_I686,
+			      __mempcpy_chk_i686)
+# if !SUPPORT_I686
+	      IFUNC_IMPL_ADD (array, i, __mempcpy_chk, HAS_I586,
+			      __mempcpy_chk_i586)
+#  if !SUPPORT_I586
 	      IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
-			      __mempcpy_chk_ia32))
+			      __mempcpy_chk_i386)
+#  endif
+# endif
+	      )
 
   /* Support sysdeps/i386/i686/multiarch/mempcpy.S.  */
   IFUNC_IMPL (i, name, mempcpy,
@@ -383,8 +422,16 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 			      __mempcpy_ssse3)
 	      IFUNC_IMPL_ADD (array, i, mempcpy, HAS_CPU_FEATURE (SSE2),
 			      __mempcpy_sse2_unaligned)
-	      IFUNC_IMPL_ADD (array, i, mempcpy, 1, __mempcpy_ia32))
+	      IFUNC_IMPL_ADD (array, i, mempcpy, HAS_I686, __mempcpy_i686)
+# if !SUPPORT_I686
+	      IFUNC_IMPL_ADD (array, i, mempcpy, HAS_I586, __mempcpy_i586)
+#  if !SUPPORT_I586
+	      IFUNC_IMPL_ADD (array, i, mempcpy, 1, __mempcpy_i386)
+#  endif
+# endif
+	      )
 
+#if 0
   /* Support sysdeps/i386/i686/multiarch/strlen.S.  */
   IFUNC_IMPL (i, name, strlen,
 	      IFUNC_IMPL_ADD (array, i, strlen, HAS_CPU_FEATURE (SSE2),
diff --git a/sysdeps/i386/multiarch/memcpy-i386.S b/sysdeps/i386/multiarch/memcpy-i386.S
new file mode 100644
index 0000000000..b3f022d279
--- /dev/null
+++ b/sysdeps/i386/multiarch/memcpy-i386.S
@@ -0,0 +1,14 @@
+#ifdef SHARED
+# include <init-arch.h>
+# if !SUPPORT_I586 && !SUPPORT_I686
+#  define memcpy __memcpy_i386
+#  define __memcpy_chk __memcpy_chk_i386
+#  undef libc_hidden_builtin_def
+#  define libc_hidden_builtin_def(name)
+#  include <sysdeps/i386/memcpy.S>
+
+	.globl __GI_memcpy
+	.hidden __GI_memcpy
+	__GI_memcpy = __memcpy_i386
+# endif
+#endif
diff --git a/sysdeps/i386/multiarch/memcpy-i586.S b/sysdeps/i386/multiarch/memcpy-i586.S
new file mode 100644
index 0000000000..35162d219b
--- /dev/null
+++ b/sysdeps/i386/multiarch/memcpy-i586.S
@@ -0,0 +1,16 @@
+#ifdef SHARED
+# include <init-arch.h>
+# if !SUPPORT_I686
+#  define memcpy __memcpy_i586
+#  define __memcpy_chk __memcpy_chk_i586
+#  undef libc_hidden_builtin_def
+#  define libc_hidden_builtin_def(name)
+#  include <sysdeps/i386/i586/memcpy.S>
+
+#  if SUPPORT_I586
+	.globl __GI_memcpy
+	.hidden __GI_memcpy
+	__GI_memcpy = __memcpy_i586
+#  endif
+# endif
+#endif
diff --git a/sysdeps/i386/multiarch/memcpy-i686.S b/sysdeps/i386/multiarch/memcpy-i686.S
new file mode 100644
index 0000000000..c0bf4a433d
--- /dev/null
+++ b/sysdeps/i386/multiarch/memcpy-i686.S
@@ -0,0 +1,14 @@
+#ifdef SHARED
+# include <init-arch.h>
+# define memcpy __memcpy_i686
+# define __memcpy_chk __memcpy_chk_i686
+# undef libc_hidden_builtin_def
+# define libc_hidden_builtin_def(name)
+# include <sysdeps/i386/i686/memcpy.S>
+
+# if SUPPORT_I686
+	.globl __GI_memcpy
+	.hidden __GI_memcpy
+	__GI_memcpy = __memcpy_i686
+# endif
+#endif
diff --git a/sysdeps/i386/i686/multiarch/memcpy-sse2-unaligned.S b/sysdeps/i386/multiarch/memcpy-sse2-unaligned.S
index 8215c70b15..8215c70b15 100644
--- a/sysdeps/i386/i686/multiarch/memcpy-sse2-unaligned.S
+++ b/sysdeps/i386/multiarch/memcpy-sse2-unaligned.S
diff --git a/sysdeps/i386/i686/multiarch/memcpy-ssse3-rep.S b/sysdeps/i386/multiarch/memcpy-ssse3-rep.S
index 08d877c03a..08d877c03a 100644
--- a/sysdeps/i386/i686/multiarch/memcpy-ssse3-rep.S
+++ b/sysdeps/i386/multiarch/memcpy-ssse3-rep.S
diff --git a/sysdeps/i386/i686/multiarch/memcpy-ssse3.S b/sysdeps/i386/multiarch/memcpy-ssse3.S
index 27ab6a2c3e..27ab6a2c3e 100644
--- a/sysdeps/i386/i686/multiarch/memcpy-ssse3.S
+++ b/sysdeps/i386/multiarch/memcpy-ssse3.S
diff --git a/sysdeps/i386/multiarch/memcpy.c b/sysdeps/i386/multiarch/memcpy.c
new file mode 100644
index 0000000000..bb3f84a39b
--- /dev/null
+++ b/sysdeps/i386/multiarch/memcpy.c
@@ -0,0 +1,69 @@
+/* Multiple versions of memcpy.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2015 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in lib and for
+   DSO.  In static binaries we need memcpy before the initialization
+   happened.  */
+#if defined SHARED && IS_IN (libc)
+/* Redefine memcpy so that the compiler won't complain about the type
+   mismatch with the IFUNC selector in strong_alias, below.  */
+# undef memcpy
+# define memcpy __redirect_memcpy
+# include <string.h>
+# undef memcpy
+
+# include <init-arch.h>
+
+extern __typeof (__redirect_memcpy) __memcpy_i386 attribute_hidden;
+extern __typeof (__redirect_memcpy) __memcpy_i586 attribute_hidden;
+extern __typeof (__redirect_memcpy) __memcpy_i686 attribute_hidden;
+extern __typeof (__redirect_memcpy) __memcpy_sse2_unaligned attribute_hidden;
+extern __typeof (__redirect_memcpy) __memcpy_ssse3 attribute_hidden;
+extern __typeof (__redirect_memcpy) __memcpy_ssse3_rep attribute_hidden;
+
+/* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle
+   ifunc symbol properly.  */
+extern __typeof (__redirect_memcpy) memcpy;
+extern void *memcpy_ifunc (void) __asm__ ("memcpy");
+
+void *
+memcpy_ifunc (void)
+{
+  if (HAS_CPU_FEATURE (SSE2))
+    {
+      if (HAS_ARCH_FEATURE (Fast_Unaligned_Load))
+	return __memcpy_sse2_unaligned;
+      else if (HAS_CPU_FEATURE (SSSE3))
+	{
+	  if (HAS_ARCH_FEATURE (Fast_Rep_String))
+	    return __memcpy_ssse3_rep;
+	  else
+	    return __memcpy_ssse3;
+	}
+    }
+
+  if (HAS_I686)
+    return __memcpy_i686;
+  else if (HAS_I586)
+    return __memcpy_i586;
+  else
+    return __memcpy_i386;
+}
+__asm__ (".type memcpy, %gnu_indirect_function");
+#endif
diff --git a/sysdeps/i386/i686/multiarch/memcpy_chk.S b/sysdeps/i386/multiarch/memcpy_chk.S
index 3bbd921555..238ab1d557 100644
--- a/sysdeps/i386/i686/multiarch/memcpy_chk.S
+++ b/sysdeps/i386/multiarch/memcpy_chk.S
@@ -18,19 +18,29 @@
    License along with the GNU C Library; if not, see
    <http://www.gnu.org/licenses/>.  */
 
-#include <sysdep.h>
-#include <init-arch.h>
-
 /* Define multiple versions only for the definition in lib and for
    DSO.  There are no multiarch memcpy functions for static binaries.
  */
 #if IS_IN (libc)
+# include <sysdep.h>
+# include <init-arch.h>
+
 # ifdef SHARED
 	.text
 ENTRY(__memcpy_chk)
 	.type	__memcpy_chk, @gnu_indirect_function
 	LOAD_GOT_AND_RTLD_GLOBAL_RO
-	LOAD_FUNC_GOT_EAX (__memcpy_chk_ia32)
+# if !SUPPORT_I686
+#  if !SUPPORT_I586
+	LOAD_FUNC_GOT_EAX (__memcpy_chk_i386)
+	HAS_ARCH_FEATURE (I586)
+	jz	2f
+#  endif
+	LOAD_FUNC_GOT_EAX (__memcpy_chk_i586)
+	HAS_ARCH_FEATURE (I686)
+	jz	2f
+# endif
+	LOAD_FUNC_GOT_EAX (__memcpy_chk_i686)
 	HAS_CPU_FEATURE (SSE2)
 	jz	2f
 	LOAD_FUNC_GOT_EAX (__memcpy_chk_sse2_unaligned)
@@ -45,6 +55,6 @@ ENTRY(__memcpy_chk)
 2:	ret
 END(__memcpy_chk)
 # else
-#  include "../memcpy_chk.S"
+#  include <sysdeps/i386/memcpy_chk.S>
 # endif
 #endif
diff --git a/sysdeps/i386/multiarch/memmove-i386.S b/sysdeps/i386/multiarch/memmove-i386.S
new file mode 100644
index 0000000000..d19cb1f42c
--- /dev/null
+++ b/sysdeps/i386/multiarch/memmove-i386.S
@@ -0,0 +1,14 @@
+#ifdef SHARED
+# include <init-arch.h>
+# if !SUPPORT_I686
+#  define memmove __memmove_i386
+#  define __memmove_chk __memmove_chk_i386
+#  undef libc_hidden_builtin_def
+#  define libc_hidden_builtin_def(name)
+#  include <sysdeps/i386/memmove.S>
+
+	.globl __GI_memmove
+	.hidden __GI_memmove
+	__GI_memmove = __memmove_i386
+# endif
+#endif
diff --git a/sysdeps/i386/multiarch/memmove-i686.S b/sysdeps/i386/multiarch/memmove-i686.S
new file mode 100644
index 0000000000..442fe4f7a3
--- /dev/null
+++ b/sysdeps/i386/multiarch/memmove-i686.S
@@ -0,0 +1,14 @@
+#ifdef SHARED
+# include <init-arch.h>
+# define memmove __memmove_i686
+# define __memmove_chk __memmove_chk_i686
+# undef libc_hidden_builtin_def
+# define libc_hidden_builtin_def(name)
+# include <sysdeps/i386/i686/memmove.S>
+
+# if SUPPORT_I686
+	.globl __GI_memmove
+	.hidden __GI_memmvoe
+	__GI_memmove = __memmove_i686
+# endif
+#endif
diff --git a/sysdeps/i386/i686/multiarch/memmove-sse2-unaligned.S b/sysdeps/i386/multiarch/memmove-sse2-unaligned.S
index 3873594cb2..3873594cb2 100644
--- a/sysdeps/i386/i686/multiarch/memmove-sse2-unaligned.S
+++ b/sysdeps/i386/multiarch/memmove-sse2-unaligned.S
diff --git a/sysdeps/i386/i686/multiarch/memmove-ssse3-rep.S b/sysdeps/i386/multiarch/memmove-ssse3-rep.S
index d202fc4a13..d202fc4a13 100644
--- a/sysdeps/i386/i686/multiarch/memmove-ssse3-rep.S
+++ b/sysdeps/i386/multiarch/memmove-ssse3-rep.S
diff --git a/sysdeps/i386/i686/multiarch/memmove-ssse3.S b/sysdeps/i386/multiarch/memmove-ssse3.S
index 295430b1ef..295430b1ef 100644
--- a/sysdeps/i386/i686/multiarch/memmove-ssse3.S
+++ b/sysdeps/i386/multiarch/memmove-ssse3.S
diff --git a/sysdeps/i386/multiarch/memmove.c b/sysdeps/i386/multiarch/memmove.c
new file mode 100644
index 0000000000..6082844868
--- /dev/null
+++ b/sysdeps/i386/multiarch/memmove.c
@@ -0,0 +1,66 @@
+/* Multiple versions of memmove.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2015 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in lib and for
+   DSO.  In static binaries we need memmove before the initialization
+   happened.  */
+#if defined SHARED && IS_IN (libc)
+/* Redefine memmove so that the compiler won't complain about the type
+   mismatch with the IFUNC selector in strong_alias, below.  */
+# undef memmove
+# define memmove __redirect_memmove
+# include <string.h>
+# undef memmove
+
+# include <init-arch.h>
+
+extern __typeof (__redirect_memmove) __memmove_i386 attribute_hidden;
+extern __typeof (__redirect_memmove) __memmove_i686 attribute_hidden;
+extern __typeof (__redirect_memmove) __memmove_sse2_unaligned attribute_hidden;
+extern __typeof (__redirect_memmove) __memmove_ssse3 attribute_hidden;
+extern __typeof (__redirect_memmove) __memmove_ssse3_rep attribute_hidden;
+
+/* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle
+   ifunc symbol properly.  */
+extern __typeof (__redirect_memmove) memmove;
+extern void *memmove_ifunc (void) __asm__ ("memmove");
+
+void *
+memmove_ifunc (void)
+{
+  if (HAS_CPU_FEATURE (SSE2))
+    {
+      if (HAS_ARCH_FEATURE (Fast_Unaligned_Load))
+	return __memmove_sse2_unaligned;
+      else if (HAS_CPU_FEATURE (SSSE3))
+	{
+	  if (HAS_ARCH_FEATURE (Fast_Rep_String))
+	    return __memmove_ssse3_rep;
+	  else
+	    return __memmove_ssse3;
+	}
+    }
+
+  if (HAS_I686)
+    return __memmove_i686;
+  else
+    return __memmove_i386;
+}
+__asm__ (".type memmove, %gnu_indirect_function");
+#endif
diff --git a/sysdeps/i386/i686/multiarch/memmove_chk.S b/sysdeps/i386/multiarch/memmove_chk.S
index b17f6edbdc..15e47d58db 100644
--- a/sysdeps/i386/i686/multiarch/memmove_chk.S
+++ b/sysdeps/i386/multiarch/memmove_chk.S
@@ -18,16 +18,21 @@
    License along with the GNU C Library; if not, see
    <http://www.gnu.org/licenses/>.  */
 
-#include <sysdep.h>
-#include <init-arch.h>
-
 /* Define multiple versions only for the definition in lib.  */
 #if IS_IN (libc)
+# include <sysdep.h>
+# include <init-arch.h>
+
 	.text
 ENTRY(__memmove_chk)
 	.type	__memmove_chk, @gnu_indirect_function
 	LOAD_GOT_AND_RTLD_GLOBAL_RO
-	LOAD_FUNC_GOT_EAX (__memmove_chk_ia32)
+# if !SUPPORT_I686
+	LOAD_FUNC_GOT_EAX (__memmove_chk_i386)
+	HAS_ARCH_FEATURE (I686)
+	jz	2f
+# endif
+	LOAD_FUNC_GOT_EAX (__memmove_chk_i686)
 	HAS_CPU_FEATURE (SSE2)
 	jz	2f
 	LOAD_FUNC_GOT_EAX (__memmove_chk_sse2_unaligned)
@@ -79,16 +84,30 @@ __memmove_chk_ssse3_rep:
 	cfi_endproc
 	.size __memmove_chk_ssse3_rep, .-__memmove_chk_ssse3_rep
 
-	.type __memmove_chk_ia32, @function
+	.type __memmove_chk_i686, @function
+	.p2align 4;
+__memmove_chk_i686:
+	cfi_startproc
+	CALL_MCOUNT
+	movl	12(%esp), %eax
+	cmpl	%eax, 16(%esp)
+	jb	__chk_fail
+	jmp	__memmove_i686
+	cfi_endproc
+	.size __memmove_chk_i686, .-__memmove_chk_i686
+
+#  if !SUPPORT_I686
+	.type __memmove_chk_i386, @function
 	.p2align 4;
-__memmove_chk_ia32:
+__memmove_chk_i386:
 	cfi_startproc
 	CALL_MCOUNT
 	movl	12(%esp), %eax
 	cmpl	%eax, 16(%esp)
 	jb	__chk_fail
-	jmp	__memmove_ia32
+	jmp	__memmove_i386
 	cfi_endproc
-	.size __memmove_chk_ia32, .-__memmove_chk_ia32
+	.size __memmove_chk_i386, .-__memmove_chk_i386
+#  endif
 # endif
 #endif
diff --git a/sysdeps/i386/multiarch/mempcpy-i386.S b/sysdeps/i386/multiarch/mempcpy-i386.S
new file mode 100644
index 0000000000..8070f8d21b
--- /dev/null
+++ b/sysdeps/i386/multiarch/mempcpy-i386.S
@@ -0,0 +1,21 @@
+#ifdef SHARED
+# include <init-arch.h>
+# if !SUPPORT_I586 && !SUPPORT_I686
+#  define __mempcpy __mempcpy_i386
+#  define __mempcpy_chk __mempcpy_chk_i386
+#  undef libc_hidden_builtin_def
+#  define libc_hidden_builtin_def(name)
+#  undef libc_hidden_def
+#  define libc_hidden_def(name)
+#  undef weak_alias
+#  define weak_alias(name, aliasname)
+#  include <sysdeps/i386/mempcpy.S>
+
+	.globl __GI_mempcpy
+	.hidden __GI_mempcpy
+	__GI_mempcpy = __mempcpy_i386
+	.globl __GI___mempcpy
+	.hidden __GI___mempcpy 
+	__GI___mempcpy = __mempcpy_i386
+# endif
+#endif
diff --git a/sysdeps/i386/multiarch/mempcpy-i586.S b/sysdeps/i386/multiarch/mempcpy-i586.S
new file mode 100644
index 0000000000..408ae5ee99
--- /dev/null
+++ b/sysdeps/i386/multiarch/mempcpy-i586.S
@@ -0,0 +1,23 @@
+#ifdef SHARED
+# include <init-arch.h>
+# if !SUPPORT_I686
+#  define __mempcpy __mempcpy_i586
+#  define __mempcpy_chk __mempcpy_chk_i586
+#  undef libc_hidden_builtin_def
+#  define libc_hidden_builtin_def(name)
+#  undef libc_hidden_def
+#  define libc_hidden_def(name)
+#  undef weak_alias
+#  define weak_alias(name, aliasname)
+#  include <sysdeps/i386/i586/mempcpy.S>
+
+#  if SUPPORT_I586
+	.globl __GI_mempcpy
+	.hidden __GI_mempcpy
+	__GI_mempcpy = __mempcpy_i586
+	.globl __GI___mempcpy
+	.hidden __GI___mempcpy 
+	__GI___mempcpy = __mempcpy_i586
+#  endif
+# endif
+#endif
diff --git a/sysdeps/i386/multiarch/mempcpy-i686.S b/sysdeps/i386/multiarch/mempcpy-i686.S
new file mode 100644
index 0000000000..f27c7a9234
--- /dev/null
+++ b/sysdeps/i386/multiarch/mempcpy-i686.S
@@ -0,0 +1,21 @@
+#ifdef SHARED
+# include <init-arch.h>
+# define __mempcpy __mempcpy_i686
+# define __mempcpy_chk __mempcpy_chk_i686
+# undef libc_hidden_builtin_def
+# define libc_hidden_builtin_def(name)
+# undef libc_hidden_def
+# define libc_hidden_def(name)
+# undef weak_alias
+# define weak_alias(name, aliasname)
+# include <sysdeps/i386/i686/mempcpy.S>
+
+# if SUPPORT_I686
+	.globl __GI_mempcpy
+	.hidden __GI_mempcpy
+	__GI_mempcpy = __mempcpy_i686
+	.globl __GI___mempcpy
+	.hidden __GI___mempcpy 
+	__GI___mempcpy = __mempcpy_i686
+# endif
+#endif
diff --git a/sysdeps/i386/i686/multiarch/mempcpy-sse2-unaligned.S b/sysdeps/i386/multiarch/mempcpy-sse2-unaligned.S
index a1cea50771..a1cea50771 100644
--- a/sysdeps/i386/i686/multiarch/mempcpy-sse2-unaligned.S
+++ b/sysdeps/i386/multiarch/mempcpy-sse2-unaligned.S
diff --git a/sysdeps/i386/i686/multiarch/mempcpy-ssse3-rep.S b/sysdeps/i386/multiarch/mempcpy-ssse3-rep.S
index 5357b33e18..5357b33e18 100644
--- a/sysdeps/i386/i686/multiarch/mempcpy-ssse3-rep.S
+++ b/sysdeps/i386/multiarch/mempcpy-ssse3-rep.S
diff --git a/sysdeps/i386/i686/multiarch/mempcpy-ssse3.S b/sysdeps/i386/multiarch/mempcpy-ssse3.S
index 822d98e954..822d98e954 100644
--- a/sysdeps/i386/i686/multiarch/mempcpy-ssse3.S
+++ b/sysdeps/i386/multiarch/mempcpy-ssse3.S
diff --git a/sysdeps/i386/multiarch/mempcpy.c b/sysdeps/i386/multiarch/mempcpy.c
new file mode 100644
index 0000000000..150f4076c4
--- /dev/null
+++ b/sysdeps/i386/multiarch/mempcpy.c
@@ -0,0 +1,71 @@
+/* Multiple versions of mempcpy.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2015 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in lib and for
+   DSO.  In static binaries we need mempcpy before the initialization
+   happened.  */
+#if defined SHARED && IS_IN (libc)
+# define _HAVE_STRING_ARCH_mempcpy
+/* Redefine mempcpy so that the compiler won't complain about the type
+   mismatch with the IFUNC selector in strong_alias, below.  */
+# undef mempcpy
+# define mempcpy __redirect_mempcpy
+# include <string.h>
+# undef mempcpy
+
+# include <init-arch.h>
+
+extern __typeof (__redirect_mempcpy) __mempcpy_i386 attribute_hidden;
+extern __typeof (__redirect_mempcpy) __mempcpy_i586 attribute_hidden;
+extern __typeof (__redirect_mempcpy) __mempcpy_i686 attribute_hidden;
+extern __typeof (__redirect_mempcpy) __mempcpy_sse2_unaligned attribute_hidden;
+extern __typeof (__redirect_mempcpy) __mempcpy_ssse3 attribute_hidden;
+extern __typeof (__redirect_mempcpy) __mempcpy_ssse3_rep attribute_hidden;
+
+/* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle
+   ifunc symbol properly.  */
+extern __typeof (__redirect_mempcpy) __mempcpy;
+extern void *mempcpy_ifunc (void) __asm__ ("__mempcpy");
+
+void *
+mempcpy_ifunc (void)
+{
+  if (HAS_CPU_FEATURE (SSE2))
+    {
+      if (HAS_ARCH_FEATURE (Fast_Unaligned_Load))
+	return __mempcpy_sse2_unaligned;
+      else if (HAS_CPU_FEATURE (SSSE3))
+	{
+	  if (HAS_ARCH_FEATURE (Fast_Rep_String))
+	    return __mempcpy_ssse3_rep;
+	  else
+	    return __mempcpy_ssse3;
+	}
+    }
+
+  if (HAS_I686)
+    return __mempcpy_i686;
+  else if (HAS_I586)
+    return __mempcpy_i586;
+  else
+    return __mempcpy_i386;
+}
+__asm__ (".type __mempcpy, %gnu_indirect_function");
+weak_alias (__mempcpy, mempcpy)
+#endif
diff --git a/sysdeps/i386/i686/multiarch/mempcpy_chk.S b/sysdeps/i386/multiarch/mempcpy_chk.S
index 1bea6eab38..496cc0064c 100644
--- a/sysdeps/i386/i686/multiarch/mempcpy_chk.S
+++ b/sysdeps/i386/multiarch/mempcpy_chk.S
@@ -18,19 +18,29 @@
    License along with the GNU C Library; if not, see
    <http://www.gnu.org/licenses/>.  */
 
-#include <sysdep.h>
-#include <init-arch.h>
-
 /* Define multiple versions only for the definition in lib and for
    DSO.  There are no multiarch mempcpy functions for static binaries.
  */
 #if IS_IN (libc)
+# include <sysdep.h>
+# include <init-arch.h>
+
 # ifdef SHARED
 	.text
 ENTRY(__mempcpy_chk)
 	.type	__mempcpy_chk, @gnu_indirect_function
 	LOAD_GOT_AND_RTLD_GLOBAL_RO
-	LOAD_FUNC_GOT_EAX (__mempcpy_chk_ia32)
+# if !SUPPORT_I686
+#  if !SUPPORT_I586
+	LOAD_FUNC_GOT_EAX (__mempcpy_chk_i386)
+	HAS_ARCH_FEATURE (I586)
+	jz	2f
+#  endif
+	LOAD_FUNC_GOT_EAX (__mempcpy_chk_i586)
+	HAS_ARCH_FEATURE (I686)
+	jz	2f
+# endif
+	LOAD_FUNC_GOT_EAX (__mempcpy_chk_i686)
 	HAS_CPU_FEATURE (SSE2)
 	jz	2f
 	LOAD_FUNC_GOT_EAX (__mempcpy_chk_sse2_unaligned)
@@ -45,6 +55,6 @@ ENTRY(__mempcpy_chk)
 2:	ret
 END(__mempcpy_chk)
 # else
-#  include "../mempcpy_chk.S"
+#  include <sysdeps/i386/mempcpy_chk.S>
 # endif
 #endif
diff --git a/sysdeps/i386/multiarch/rtld-memmove.S b/sysdeps/i386/multiarch/rtld-memmove.S
new file mode 100644
index 0000000000..efbbd21f2e
--- /dev/null
+++ b/sysdeps/i386/multiarch/rtld-memmove.S
@@ -0,0 +1,24 @@
+/* memmove for ld.so
+   Copyright (C) 2015 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <init-arch.h>
+#if SUPPORT_I686
+# include <sysdeps/i386/i686/memmove.S>
+#else
+# include <sysdeps/i386/memmove.S>
+#endif
diff --git a/sysdeps/i386/multiarch/static-memmove.S b/sysdeps/i386/multiarch/static-memmove.S
new file mode 100644
index 0000000000..4debbbe93b
--- /dev/null
+++ b/sysdeps/i386/multiarch/static-memmove.S
@@ -0,0 +1,26 @@
+/* memmove for libc.a
+   Copyright (C) 2015 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#if !defined SHARED && IS_IN (libc)
+# include <init-arch.h>
+# if SUPPORT_I686
+#  include <sysdeps/i386/i686/memmove.S>
+# else
+#  include <sysdeps/i386/memmove.S>
+# endif
+#endif