diff options
25 files changed, 643 insertions, 240 deletions
diff --git a/ChangeLog b/ChangeLog index d2dd7d468d..f96acc89df 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,41 @@ +2017-06-21 H.J. Lu <hongjiu.lu@intel.com> + + * sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Add + strcmp-sse2, strcmp-sse4_2, strncmp-sse2, strncmp-sse4_2, + strcasecmp_l-sse2, strcasecmp_l-sse4_2, strcasecmp_l-avx, + strncase_l-sse2, strncase_l-sse4_2 and strncase_l-avx. + * sysdeps/x86_64/multiarch/ifunc-strcasecmp.h: New file. + * sysdeps/x86_64/multiarch/strcasecmp.c: Likewise. + * sysdeps/x86_64/multiarch/strcasecmp_l-avx.S: Likewise. + * sysdeps/x86_64/multiarch/strcasecmp_l-sse2.S: Likewise. + * sysdeps/x86_64/multiarch/strcasecmp_l-sse4_2.S: Likewise. + * sysdeps/x86_64/multiarch/strcasecmp_l.c: Likewise. + * sysdeps/x86_64/multiarch/strcmp-sse2.S: Likewise. + * sysdeps/x86_64/multiarch/strcmp-sse4_2.S: Likewise. + * sysdeps/x86_64/multiarch/strcmp.c: Likewise. + * sysdeps/x86_64/multiarch/strncase.c: Likewise. + * sysdeps/x86_64/multiarch/strncase_l-avx.S : Likewise. + * sysdeps/x86_64/multiarch/strncase_l-sse2.S: Likewise. + * sysdeps/x86_64/multiarch/strncase_l-sse4_2.S: Likewise. + * sysdeps/x86_64/multiarch/strncase_l.c: Likewise. + * sysdeps/x86_64/multiarch/strncmp-sse2.S: Likewise. + * sysdeps/x86_64/multiarch/strncmp-sse4_2.S: Likewise. + * sysdeps/x86_64/multiarch/strncmp.c: Likewise. + * sysdeps/x86_64/multiarch/strcasecmp_l.S: Removed. + * sysdeps/x86_64/multiarch/strcmp.S: Likewise. + * sysdeps/x86_64/multiarch/strncase_l.S: Likewise. + * sysdeps/x86_64/multiarch/strncmp.S: Likewise. + * sysdeps/x86_64/multiarch/strcmp-sse42.S: Include <sysdep.h>. + (STRCMP_SSE42): New. Defined to __strcmp_sse42 if not defined. + [USE_AS_STRCASECMP_L || USE_AS_STRNCASECMP_L]: Include + "locale-defines.h". + (UPDATE_STRNCMP_COUNTER): New. + (SECTION): Likewise. + (GLABEL): Likewise. + (LABEL): Likewise. + * sysdeps/x86_64/multiarch/strncmp-ssse3.S: Rewrite and enable + for libc.a. + 2017-06-21 Joseph Myers <joseph@codesourcery.com> [BZ #21622] diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile index 2c54c5cc23..68257c4017 100644 --- a/sysdeps/x86_64/multiarch/Makefile +++ b/sysdeps/x86_64/multiarch/Makefile @@ -4,8 +4,10 @@ endif ifeq ($(subdir),string) -sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \ - strcmp-sse2-unaligned strncmp-ssse3 \ +sysdep_routines += strncat-c stpncpy-c strncpy-c \ + strcmp-sse2 strcmp-sse2-unaligned strcmp-ssse3 \ + strcmp-sse4_2 \ + strncmp-sse2 strncmp-ssse3 strncmp-sse4_2 \ memchr-sse2 rawmemchr-sse2 memchr-avx2 rawmemchr-avx2 \ memrchr-sse2 memrchr-avx2 \ memcmp-sse2 \ @@ -14,11 +16,15 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \ memmove-ssse3 \ memcpy-ssse3-back \ memmove-ssse3-back \ - memmove-avx512-no-vzeroupper strcasecmp_l-ssse3 \ + memmove-avx512-no-vzeroupper \ + strcasecmp_l-sse2 strcasecmp_l-ssse3 \ + strcasecmp_l-sse4_2 strcasecmp_l-avx \ + strncase_l-sse2 strncase_l-ssse3 \ + strncase_l-sse4_2 strncase_l-avx \ strchr-sse2 strchrnul-sse2 strchr-avx2 strchrnul-avx2 \ strrchr-sse2 strrchr-avx2 \ strlen-sse2 strnlen-sse2 strlen-avx2 strnlen-avx2 \ - strncase_l-ssse3 strcat-ssse3 strncat-ssse3\ + strcat-ssse3 strncat-ssse3\ strcpy-sse2 stpcpy-sse2 \ strcpy-ssse3 strncpy-ssse3 stpcpy-ssse3 stpncpy-ssse3 \ strcpy-sse2-unaligned strncpy-sse2-unaligned \ diff --git a/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h b/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h new file mode 100644 index 0000000000..adecd6d8e0 --- /dev/null +++ b/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h @@ -0,0 +1,43 @@ +/* Common definition for strcasecmp famly ifunc selections. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <init-arch.h> + +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse42) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx) attribute_hidden; + +static inline void * +IFUNC_SELECTOR (void) +{ + const struct cpu_features* cpu_features = __get_cpu_features (); + + if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Usable)) + return OPTIMIZE (avx); + + if (CPU_FEATURES_CPU_P (cpu_features, SSE4_2) + && !CPU_FEATURES_ARCH_P (cpu_features, Slow_SSE4_2)) + return OPTIMIZE (sse42); + + if (CPU_FEATURES_CPU_P (cpu_features, SSSE3)) + return OPTIMIZE (ssse3); + + return OPTIMIZE (sse2); +} diff --git a/sysdeps/x86_64/multiarch/strcasecmp.c b/sysdeps/x86_64/multiarch/strcasecmp.c new file mode 100644 index 0000000000..8170bdee69 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcasecmp.c @@ -0,0 +1,39 @@ +/* Multiple versions of strcasecmp. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define strcasecmp __redirect_strcasecmp +# define __strcasecmp __redirect___strcasecmp +# include <string.h> +# undef strcasecmp +# undef __strcasecmp + +# define SYMBOL_NAME strcasecmp +# include "ifunc-strcasecmp.h" + +libc_ifunc_redirected (__redirect_strcasecmp, __strcasecmp, + IFUNC_SELECTOR ()); + +weak_alias (__strcasecmp, strcasecmp) +# ifdef SHARED +__hidden_ver1 (__strcasecmp, __GI___strcasecmp, __redirect___strcasecmp) + __attribute__ ((visibility ("hidden"))); +# endif +#endif diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l-avx.S b/sysdeps/x86_64/multiarch/strcasecmp_l-avx.S new file mode 100644 index 0000000000..60f5b50c66 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcasecmp_l-avx.S @@ -0,0 +1,22 @@ +/* strcasecmp_l optimized with AVX. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#define STRCMP_SSE42 __strcasecmp_l_avx +#define USE_AVX 1 +#define USE_AS_STRCASECMP_L +#include "strcmp-sse42.S" diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l-sse2.S b/sysdeps/x86_64/multiarch/strcasecmp_l-sse2.S new file mode 100644 index 0000000000..0b04184c5c --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcasecmp_l-sse2.S @@ -0,0 +1,23 @@ +/* strcasecmp_l optimized with SSE2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#define STRCMP __strcasecmp_l_sse2 +#define USE_AS_STRCASECMP_L +#define NO_NOLOCALE_ALIAS +#define __strcasecmp __strcasecmp_sse2 +#include <sysdeps/x86_64/strcmp.S> diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l-sse4_2.S b/sysdeps/x86_64/multiarch/strcasecmp_l-sse4_2.S new file mode 100644 index 0000000000..92a660be01 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcasecmp_l-sse4_2.S @@ -0,0 +1,21 @@ +/* strcasecmp_l optimized with SSE4.2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#define STRCMP_SSE42 __strcasecmp_l_sse42 +#define USE_AS_STRCASECMP_L +#include "strcmp-sse42.S" diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l.S b/sysdeps/x86_64/multiarch/strcasecmp_l.S deleted file mode 100644 index 49f5b9fd95..0000000000 --- a/sysdeps/x86_64/multiarch/strcasecmp_l.S +++ /dev/null @@ -1,8 +0,0 @@ -/* Multiple versions of strcasecmp and strcasecmp_l - All versions must be listed in ifunc-impl-list.c. */ -#define STRCMP __strcasecmp_l -#define USE_AS_STRCASECMP_L -#include "strcmp.S" - -weak_alias (__strcasecmp_l, strcasecmp_l) -libc_hidden_def (strcasecmp_l) diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l.c b/sysdeps/x86_64/multiarch/strcasecmp_l.c new file mode 100644 index 0000000000..f28db6cd20 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcasecmp_l.c @@ -0,0 +1,40 @@ +/* Multiple versions of strcasecmp_l. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define strcasecmp_l __redirect_strcasecmp_l +# define __strcasecmp_l __redirect___strcasecmp_l +# include <string.h> +# undef strcasecmp_l +# undef __strcasecmp_l + +# define SYMBOL_NAME strcasecmp_l +# include "ifunc-strcasecmp.h" + +libc_ifunc_redirected (__redirect_strcasecmp_l, __strcasecmp_l, + IFUNC_SELECTOR ()); + +weak_alias (__strcasecmp_l, strcasecmp_l) +# ifdef SHARED +__hidden_ver1 (__strcasecmp_l, __GI___strcasecmp_l, + __redirect___strcasecmp_l) + __attribute__ ((visibility ("hidden"))); +# endif +#endif diff --git a/sysdeps/x86_64/multiarch/strcmp-sse2.S b/sysdeps/x86_64/multiarch/strcmp-sse2.S new file mode 100644 index 0000000000..834ba3f093 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcmp-sse2.S @@ -0,0 +1,28 @@ +/* strcmp optimized with SSE2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#if IS_IN (libc) +# include <sysdep.h> + +# define STRCMP __strcmp_sse2 + +# undef libc_hidden_builtin_def +# define libc_hidden_builtin_def(strcmp) +#endif + +#include <sysdeps/x86_64/strcmp.S> diff --git a/sysdeps/x86_64/multiarch/strcmp-sse42.S b/sysdeps/x86_64/multiarch/strcmp-sse42.S index ed26d4a8fb..4aeb14e175 100644 --- a/sysdeps/x86_64/multiarch/strcmp-sse42.S +++ b/sysdeps/x86_64/multiarch/strcmp-sse42.S @@ -17,6 +17,40 @@ License along with the GNU C Library; if not, see <http://www.gnu.org/licenses/>. */ +#include <sysdep.h> + +#ifndef STRCMP_SSE42 +# define STRCMP_SSE42 __strcmp_sse42 +#endif + +#if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L +# include "locale-defines.h" +#endif + +#if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L +/* Since the counter, %r11, is unsigned, we branch to strcmp_exitz + if the new counter > the old one or is 0. */ +# define UPDATE_STRNCMP_COUNTER \ + /* calculate left number to compare */ \ + lea -16(%rcx, %r11), %r9; \ + cmp %r9, %r11; \ + jb LABEL(strcmp_exitz); \ + test %r9, %r9; \ + je LABEL(strcmp_exitz); \ + mov %r9, %r11 +#else +# define UPDATE_STRNCMP_COUNTER +#endif + +#ifdef USE_AVX +# define SECTION avx +# define GLABEL(l) l##_avx +#else +# define SECTION sse4.2 +# define GLABEL(l) l##_sse42 +#endif + +#define LABEL(l) .L##l /* We use 0x1a: _SIDD_SBYTE_OPS diff --git a/sysdeps/x86_64/multiarch/strcmp-sse4_2.S b/sysdeps/x86_64/multiarch/strcmp-sse4_2.S new file mode 100644 index 0000000000..acf0079a78 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcmp-sse4_2.S @@ -0,0 +1,21 @@ +/* strcmp optimized with SSE4.2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#if IS_IN (libc) +# include "strcmp-sse42.S" +#endif diff --git a/sysdeps/x86_64/multiarch/strcmp.S b/sysdeps/x86_64/multiarch/strcmp.S deleted file mode 100644 index 54f8f7dd44..0000000000 --- a/sysdeps/x86_64/multiarch/strcmp.S +++ /dev/null @@ -1,209 +0,0 @@ -/* Multiple versions of strcmp - Copyright (C) 2009-2017 Free Software Foundation, Inc. - Contributed by Intel Corporation. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#include <sysdep.h> -#include <init-arch.h> - -#ifdef USE_AS_STRNCMP -/* Since the counter, %r11, is unsigned, we branch to strcmp_exitz - if the new counter > the old one or is 0. */ -# define UPDATE_STRNCMP_COUNTER \ - /* calculate left number to compare */ \ - lea -16(%rcx, %r11), %r9; \ - cmp %r9, %r11; \ - jb LABEL(strcmp_exitz); \ - test %r9, %r9; \ - je LABEL(strcmp_exitz); \ - mov %r9, %r11 - -# define STRCMP_SSE42 __strncmp_sse42 -# define STRCMP_SSSE3 __strncmp_ssse3 -# define STRCMP_SSE2 __strncmp_sse2 -# define __GI_STRCMP __GI_strncmp -#elif defined USE_AS_STRCASECMP_L -# include "locale-defines.h" - -# define UPDATE_STRNCMP_COUNTER - -# define STRCMP_AVX __strcasecmp_l_avx -# define STRCMP_SSE42 __strcasecmp_l_sse42 -# define STRCMP_SSSE3 __strcasecmp_l_ssse3 -# define STRCMP_SSE2 __strcasecmp_l_sse2 -# define __GI_STRCMP __GI___strcasecmp_l -#elif defined USE_AS_STRNCASECMP_L -# include "locale-defines.h" - -/* Since the counter, %r11, is unsigned, we branch to strcmp_exitz - if the new counter > the old one or is 0. */ -# define UPDATE_STRNCMP_COUNTER \ - /* calculate left number to compare */ \ - lea -16(%rcx, %r11), %r9; \ - cmp %r9, %r11; \ - jb LABEL(strcmp_exitz); \ - test %r9, %r9; \ - je LABEL(strcmp_exitz); \ - mov %r9, %r11 - -# define STRCMP_AVX __strncasecmp_l_avx -# define STRCMP_SSE42 __strncasecmp_l_sse42 -# define STRCMP_SSSE3 __strncasecmp_l_ssse3 -# define STRCMP_SSE2 __strncasecmp_l_sse2 -# define __GI_STRCMP __GI___strncasecmp_l -#else -# define USE_AS_STRCMP -# define UPDATE_STRNCMP_COUNTER -# ifndef STRCMP -# define STRCMP strcmp -# define STRCMP_SSE42 __strcmp_sse42 -# define STRCMP_SSSE3 __strcmp_ssse3 -# define STRCMP_SSE2 __strcmp_sse2 -# define __GI_STRCMP __GI_strcmp -# endif -#endif - -/* Define multiple versions only for the definition in libc. Don't - define multiple versions for strncmp in static library since we - need strncmp before the initialization happened. */ -#if (defined SHARED || !defined USE_AS_STRNCMP) && IS_IN (libc) - .text -ENTRY(STRCMP) - .type STRCMP, @gnu_indirect_function - LOAD_RTLD_GLOBAL_RO_RDX -#ifdef USE_AS_STRCMP - leaq __strcmp_sse2_unaligned(%rip), %rax - HAS_ARCH_FEATURE (Fast_Unaligned_Load) - jnz 3f -#else - HAS_ARCH_FEATURE (Slow_SSE4_2) - jnz 2f - leaq STRCMP_SSE42(%rip), %rax - HAS_CPU_FEATURE (SSE4_2) - jnz 3f -#endif -2: leaq STRCMP_SSSE3(%rip), %rax - HAS_CPU_FEATURE (SSSE3) - jnz 3f - leaq STRCMP_SSE2(%rip), %rax -3: ret -END(STRCMP) - -# ifdef USE_AS_STRCASECMP_L -ENTRY(__strcasecmp) - .type __strcasecmp, @gnu_indirect_function - LOAD_RTLD_GLOBAL_RO_RDX - leaq __strcasecmp_avx(%rip), %rax - HAS_ARCH_FEATURE (AVX_Usable) - jnz 3f - HAS_ARCH_FEATURE (Slow_SSE4_2) - jnz 2f - leaq __strcasecmp_sse42(%rip), %rax - HAS_CPU_FEATURE (SSE4_2) - jnz 3f -2: leaq __strcasecmp_ssse3(%rip), %rax - HAS_CPU_FEATURE (SSSE3) - jnz 3f - leaq __strcasecmp_sse2(%rip), %rax -3: ret -END(__strcasecmp) -weak_alias (__strcasecmp, strcasecmp) -# endif -# ifdef USE_AS_STRNCASECMP_L -ENTRY(__strncasecmp) - .type __strncasecmp, @gnu_indirect_function - LOAD_RTLD_GLOBAL_RO_RDX - leaq __strncasecmp_avx(%rip), %rax - HAS_ARCH_FEATURE (AVX_Usable) - jnz 3f - HAS_ARCH_FEATURE (Slow_SSE4_2) - jnz 2f - leaq __strncasecmp_sse42(%rip), %rax - HAS_CPU_FEATURE (SSE4_2) - jnz 3f -2: leaq __strncasecmp_ssse3(%rip), %rax - HAS_CPU_FEATURE (SSSE3) - jnz 3f - leaq __strncasecmp_sse2(%rip), %rax -3: ret -END(__strncasecmp) -weak_alias (__strncasecmp, strncasecmp) -# endif - -# undef LABEL -# define LABEL(l) .L##l##_sse42 -# define GLABEL(l) l##_sse42 -# define SECTION sse4.2 -# include "strcmp-sse42.S" - - -# if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L -# define LABEL(l) .L##l##_avx -# define GLABEL(l) l##_avx -# define USE_AVX 1 -# undef STRCMP_SSE42 -# define STRCMP_SSE42 STRCMP_AVX -# define SECTION avx -# include "strcmp-sse42.S" -# endif - - -# undef ENTRY -# define ENTRY(name) \ - .type STRCMP_SSE2, @function; \ - .align 16; \ - .globl STRCMP_SSE2; \ - .hidden STRCMP_SSE2; \ - STRCMP_SSE2: cfi_startproc; \ - CALL_MCOUNT -# undef END -# define END(name) \ - cfi_endproc; .size STRCMP_SSE2, .-STRCMP_SSE2 - -# ifdef USE_AS_STRCASECMP_L -# define ENTRY2(name) \ - .type __strcasecmp_sse2, @function; \ - .align 16; \ - .globl __strcasecmp_sse2; \ - .hidden __strcasecmp_sse2; \ - __strcasecmp_sse2: cfi_startproc; \ - CALL_MCOUNT -# define END2(name) \ - cfi_endproc; .size __strcasecmp_sse2, .-__strcasecmp_sse2 -# endif - -# ifdef USE_AS_STRNCASECMP_L -# define ENTRY2(name) \ - .type __strncasecmp_sse2, @function; \ - .align 16; \ - .globl __strncasecmp_sse2; \ - .hidden __strncasecmp_sse2; \ - __strncasecmp_sse2: cfi_startproc; \ - CALL_MCOUNT -# define END2(name) \ - cfi_endproc; .size __strncasecmp_sse2, .-__strncasecmp_sse2 -# endif - -# undef libc_hidden_builtin_def -/* It doesn't make sense to send libc-internal strcmp calls through a PLT. - The speedup we get from using SSE4.2 instruction is likely eaten away - by the indirect call in the PLT. */ -# define libc_hidden_builtin_def(name) \ - .globl __GI_STRCMP; __GI_STRCMP = STRCMP_SSE2 -#endif - -#include "../strcmp.S" diff --git a/sysdeps/x86_64/multiarch/strcmp.c b/sysdeps/x86_64/multiarch/strcmp.c new file mode 100644 index 0000000000..5c0fc5a76f --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcmp.c @@ -0,0 +1,53 @@ +/* Multiple versions of strcmp. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define strcmp __redirect_strcmp +# include <string.h> +# undef strcmp + +# define SYMBOL_NAME strcmp +# include <init-arch.h> + +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; + +static inline void * +IFUNC_SELECTOR (void) +{ + const struct cpu_features* cpu_features = __get_cpu_features (); + + if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Load)) + return OPTIMIZE (sse2_unaligned); + + if (CPU_FEATURES_CPU_P (cpu_features, SSSE3)) + return OPTIMIZE (ssse3); + + return OPTIMIZE (sse2); +} + +libc_ifunc_redirected (__redirect_strcmp, strcmp, IFUNC_SELECTOR ()); + +# ifdef SHARED +__hidden_ver1 (strcmp, __GI_strcmp, __redirect_strcmp) + __attribute__ ((visibility ("hidden"))); +# endif +#endif diff --git a/sysdeps/x86_64/multiarch/strncase.c b/sysdeps/x86_64/multiarch/strncase.c new file mode 100644 index 0000000000..4bb4c61b8e --- /dev/null +++ b/sysdeps/x86_64/multiarch/strncase.c @@ -0,0 +1,35 @@ +/* Multiple versions of strncasecmp. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define strncasecmp __redirect_strncasecmp +# define __strncasecmp __redirect___strncasecmp +# include <string.h> +# undef strncasecmp +# undef __strncasecmp + +# define SYMBOL_NAME strncasecmp +# include "ifunc-strcasecmp.h" + +libc_ifunc_redirected (__redirect_strncasecmp, __strncasecmp, + IFUNC_SELECTOR ()); + +weak_alias (__strncasecmp, strncasecmp) +#endif diff --git a/sysdeps/x86_64/multiarch/strncase_l-avx.S b/sysdeps/x86_64/multiarch/strncase_l-avx.S new file mode 100644 index 0000000000..2a109f1e9a --- /dev/null +++ b/sysdeps/x86_64/multiarch/strncase_l-avx.S @@ -0,0 +1,22 @@ +/* strncasecmp_l optimized with AVX. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#define STRCMP_SSE42 __strncasecmp_l_avx +#define USE_AVX 1 +#define USE_AS_STRNCASECMP_L +#include "strcmp-sse42.S" diff --git a/sysdeps/x86_64/multiarch/strncase_l-sse2.S b/sysdeps/x86_64/multiarch/strncase_l-sse2.S new file mode 100644 index 0000000000..6cd3d91fe1 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strncase_l-sse2.S @@ -0,0 +1,23 @@ +/* strncasecmp_l optimized with SSE2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#define STRCMP __strncasecmp_l_sse2 +#define NO_NOLOCALE_ALIAS +#define USE_AS_STRNCASECMP_L +#define __strncasecmp __strncasecmp_sse2 +#include <sysdeps/x86_64/strcmp.S> diff --git a/sysdeps/x86_64/multiarch/strncase_l-sse4_2.S b/sysdeps/x86_64/multiarch/strncase_l-sse4_2.S new file mode 100644 index 0000000000..f381dcf7e0 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strncase_l-sse4_2.S @@ -0,0 +1,21 @@ +/* strncasecmp_l optimized with SSE4.2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#define STRCMP_SSE42 __strncasecmp_l_sse42 +#define USE_AS_STRNCASECMP_L +#include "strcmp-sse42.S" diff --git a/sysdeps/x86_64/multiarch/strncase_l.S b/sysdeps/x86_64/multiarch/strncase_l.S deleted file mode 100644 index 9c0149788e..0000000000 --- a/sysdeps/x86_64/multiarch/strncase_l.S +++ /dev/null @@ -1,8 +0,0 @@ -/* Multiple versions of strncasecmp and strncasecmp_l - All versions must be listed in ifunc-impl-list.c. */ -#define STRCMP __strncasecmp_l -#define USE_AS_STRNCASECMP_L -#include "strcmp.S" - -weak_alias (__strncasecmp_l, strncasecmp_l) -libc_hidden_def (strncasecmp_l) diff --git a/sysdeps/x86_64/multiarch/strncase_l.c b/sysdeps/x86_64/multiarch/strncase_l.c new file mode 100644 index 0000000000..e6ca6bb5cc --- /dev/null +++ b/sysdeps/x86_64/multiarch/strncase_l.c @@ -0,0 +1,40 @@ +/* Multiple versions of strncasecmp_l. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define strncasecmp_l __redirect_strncasecmp_l +# define __strncasecmp_l __redirect___strncasecmp_l +# include <string.h> +# undef strncasecmp_l +# undef __strncasecmp_l + +# define SYMBOL_NAME strncasecmp_l +# include "ifunc-strcasecmp.h" + +libc_ifunc_redirected (__redirect_strncasecmp_l, __strncasecmp_l, + IFUNC_SELECTOR ()); + +weak_alias (__strncasecmp_l, strncasecmp_l) +# ifdef SHARED +__hidden_ver1 (__strncasecmp_l, __GI___strncasecmp_l, + __redirect___strncasecmp_l) + __attribute__ ((visibility ("hidden"))); +# endif +#endif diff --git a/sysdeps/x86_64/multiarch/strncmp-sse2.S b/sysdeps/x86_64/multiarch/strncmp-sse2.S new file mode 100644 index 0000000000..f16d33dc01 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strncmp-sse2.S @@ -0,0 +1,27 @@ +/* strcmp optimized with SSE2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> + +#define STRCMP __strncmp_sse2 + +#undef libc_hidden_builtin_def +#define libc_hidden_builtin_def(strcmp) + +#define USE_AS_STRNCMP +#include <sysdeps/x86_64/strcmp.S> diff --git a/sysdeps/x86_64/multiarch/strncmp-sse4_2.S b/sysdeps/x86_64/multiarch/strncmp-sse4_2.S new file mode 100644 index 0000000000..21a03f2119 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strncmp-sse4_2.S @@ -0,0 +1,21 @@ +/* strncmp optimized with SSE4.2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#define STRCMP_SSE42 __strncmp_sse42 +#define USE_AS_STRNCMP +#include "strcmp-sse42.S" diff --git a/sysdeps/x86_64/multiarch/strncmp-ssse3.S b/sysdeps/x86_64/multiarch/strncmp-ssse3.S index 96380a46be..254cc74ba2 100644 --- a/sysdeps/x86_64/multiarch/strncmp-ssse3.S +++ b/sysdeps/x86_64/multiarch/strncmp-ssse3.S @@ -1,6 +1,28 @@ -#ifdef SHARED -# define USE_SSSE3 1 -# define STRCMP __strncmp_ssse3 -# define USE_AS_STRNCMP -# include "../strcmp.S" -#endif +/* strcmp optimized with SSSE3. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> + +#define STRCMP __strncmp_ssse3 + +#undef libc_hidden_builtin_def +#define libc_hidden_builtin_def(strcmp) + +#define USE_SSSE3 1 +#define USE_AS_STRNCMP +#include <sysdeps/x86_64/strcmp.S> diff --git a/sysdeps/x86_64/multiarch/strncmp.S b/sysdeps/x86_64/multiarch/strncmp.S deleted file mode 100644 index fd5eb1397c..0000000000 --- a/sysdeps/x86_64/multiarch/strncmp.S +++ /dev/null @@ -1,5 +0,0 @@ -/* Multiple versions of strncmp - All versions must be listed in ifunc-impl-list.c. */ -#define STRCMP strncmp -#define USE_AS_STRNCMP -#include "strcmp.S" diff --git a/sysdeps/x86_64/multiarch/strncmp.c b/sysdeps/x86_64/multiarch/strncmp.c new file mode 100644 index 0000000000..1457d4237d --- /dev/null +++ b/sysdeps/x86_64/multiarch/strncmp.c @@ -0,0 +1,54 @@ +/* Multiple versions of strncmp. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define strncmp __redirect_strncmp +# include <string.h> +# undef strncmp + +# define SYMBOL_NAME strncmp +# include <init-arch.h> + +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse42) attribute_hidden; + +static inline void * +IFUNC_SELECTOR (void) +{ + const struct cpu_features* cpu_features = __get_cpu_features (); + + if (CPU_FEATURES_CPU_P (cpu_features, SSE4_2) + && !CPU_FEATURES_ARCH_P (cpu_features, Slow_SSE4_2)) + return OPTIMIZE (sse42); + + if (CPU_FEATURES_CPU_P (cpu_features, SSSE3)) + return OPTIMIZE (ssse3); + + return OPTIMIZE (sse2); +} + +libc_ifunc_redirected (__redirect_strncmp, strncmp, IFUNC_SELECTOR ()); + +# ifdef SHARED +__hidden_ver1 (strncmp, __GI_strncmp, __redirect_strncmp) + __attribute__ ((visibility ("hidden"))); +# endif +#endif |