From 135407f43175899db98e9881daffed856a38042c Mon Sep 17 00:00:00 2001 From: dengjianbo Date: Tue, 8 Aug 2023 14:15:44 +0800 Subject: Loongarch: Add ifunc support and add different versions of strlen strlen-lasx is implemeted by LASX simd instructions(256bit) strlen-lsx is implemeted by LSX simd instructions(128bit) strlen-align is implemented by LA basic instructions and never use unaligned memory acess --- sysdeps/loongarch/lp64/multiarch/Makefile | 7 ++ sysdeps/loongarch/lp64/multiarch/ifunc-impl-list.c | 41 +++++++++ sysdeps/loongarch/lp64/multiarch/ifunc-strlen.h | 40 +++++++++ sysdeps/loongarch/lp64/multiarch/strlen-aligned.S | 100 +++++++++++++++++++++ sysdeps/loongarch/lp64/multiarch/strlen-lasx.S | 63 +++++++++++++ sysdeps/loongarch/lp64/multiarch/strlen-lsx.S | 71 +++++++++++++++ sysdeps/loongarch/lp64/multiarch/strlen.c | 37 ++++++++ 7 files changed, 359 insertions(+) create mode 100644 sysdeps/loongarch/lp64/multiarch/Makefile create mode 100644 sysdeps/loongarch/lp64/multiarch/ifunc-impl-list.c create mode 100644 sysdeps/loongarch/lp64/multiarch/ifunc-strlen.h create mode 100644 sysdeps/loongarch/lp64/multiarch/strlen-aligned.S create mode 100644 sysdeps/loongarch/lp64/multiarch/strlen-lasx.S create mode 100644 sysdeps/loongarch/lp64/multiarch/strlen-lsx.S create mode 100644 sysdeps/loongarch/lp64/multiarch/strlen.c (limited to 'sysdeps/loongarch/lp64/multiarch') diff --git a/sysdeps/loongarch/lp64/multiarch/Makefile b/sysdeps/loongarch/lp64/multiarch/Makefile new file mode 100644 index 0000000000..76c506c966 --- /dev/null +++ b/sysdeps/loongarch/lp64/multiarch/Makefile @@ -0,0 +1,7 @@ +ifeq ($(subdir),string) +sysdep_routines += \ + strlen-aligned \ + strlen-lsx \ + strlen-lasx \ +# sysdep_routines +endif diff --git a/sysdeps/loongarch/lp64/multiarch/ifunc-impl-list.c b/sysdeps/loongarch/lp64/multiarch/ifunc-impl-list.c new file mode 100644 index 0000000000..1a2a576fcd --- /dev/null +++ b/sysdeps/loongarch/lp64/multiarch/ifunc-impl-list.c @@ -0,0 +1,41 @@ +/* Enumerate available IFUNC implementations of a function LoongArch64 version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include +#include +#include +#include +#include +#include + +size_t +__libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + size_t max) +{ + + size_t i = max; + + IFUNC_IMPL (i, name, strlen, +#if !defined __loongarch_soft_float + IFUNC_IMPL_ADD (array, i, strlen, SUPPORT_LASX, __strlen_lasx) + IFUNC_IMPL_ADD (array, i, strlen, SUPPORT_LSX, __strlen_lsx) +#endif + IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_aligned) + ) + return i; +} diff --git a/sysdeps/loongarch/lp64/multiarch/ifunc-strlen.h b/sysdeps/loongarch/lp64/multiarch/ifunc-strlen.h new file mode 100644 index 0000000000..6258bb76c3 --- /dev/null +++ b/sysdeps/loongarch/lp64/multiarch/ifunc-strlen.h @@ -0,0 +1,40 @@ +/* Common definition for strlen ifunc selections. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include +#include + +#if !defined __loongarch_soft_float +extern __typeof (REDIRECT_NAME) OPTIMIZE (lasx) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (lsx) attribute_hidden; +#endif +extern __typeof (REDIRECT_NAME) OPTIMIZE (aligned) attribute_hidden; + +static inline void * +IFUNC_SELECTOR (void) +{ +#if !defined __loongarch_soft_float + if (SUPPORT_LASX) + return OPTIMIZE (lasx); + else if (SUPPORT_LSX) + return OPTIMIZE (lsx); + else +#endif + return OPTIMIZE (aligned); +} diff --git a/sysdeps/loongarch/lp64/multiarch/strlen-aligned.S b/sysdeps/loongarch/lp64/multiarch/strlen-aligned.S new file mode 100644 index 0000000000..e9e1d2fc04 --- /dev/null +++ b/sysdeps/loongarch/lp64/multiarch/strlen-aligned.S @@ -0,0 +1,100 @@ +/* Optimized strlen implementation using basic Loongarch instructions. + Copyright (C) 2023 Free Software Foundation, Inc. + + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library. If not, see + . */ + +#include +#include +#include + +#if IS_IN (libc) +# define STRLEN __strlen_aligned +#else +# define STRLEN strlen +#endif + +LEAF(STRLEN, 6) + move a1, a0 + bstrins.d a0, zero, 2, 0 + lu12i.w a2, 0x01010 + li.w t0, -1 + + ld.d t2, a0, 0 + andi t1, a1, 0x7 + ori a2, a2, 0x101 + slli.d t1, t1, 3 + + bstrins.d a2, a2, 63, 32 + sll.d t1, t0, t1 + slli.d t3, a2, 7 + nor a3, zero, t3 + + orn t2, t2, t1 + sub.d t0, t2, a2 + nor t1, t2, a3 + and t0, t0, t1 + + + bnez t0, L(count_pos) + addi.d a0, a0, 8 +L(loop_16_7bit): + ld.d t2, a0, 0 + sub.d t1, t2, a2 + + and t0, t1, t3 + bnez t0, L(more_check) + ld.d t2, a0, 8 + sub.d t1, t2, a2 + + and t0, t1, t3 + addi.d a0, a0, 16 + beqz t0, L(loop_16_7bit) + addi.d a0, a0, -8 + +L(more_check): + nor t0, t2, a3 + and t0, t1, t0 + bnez t0, L(count_pos) + addi.d a0, a0, 8 + + +L(loop_16_8bit): + ld.d t2, a0, 0 + sub.d t1, t2, a2 + nor t0, t2, a3 + and t0, t0, t1 + + bnez t0, L(count_pos) + ld.d t2, a0, 8 + addi.d a0, a0, 16 + sub.d t1, t2, a2 + + nor t0, t2, a3 + and t0, t0, t1 + beqz t0, L(loop_16_8bit) + addi.d a0, a0, -8 + +L(count_pos): + ctz.d t1, t0 + sub.d a0, a0, a1 + srli.d t1, t1, 3 + add.d a0, a0, t1 + + jr ra +END(STRLEN) + +libc_hidden_builtin_def (STRLEN) diff --git a/sysdeps/loongarch/lp64/multiarch/strlen-lasx.S b/sysdeps/loongarch/lp64/multiarch/strlen-lasx.S new file mode 100644 index 0000000000..258c47cea0 --- /dev/null +++ b/sysdeps/loongarch/lp64/multiarch/strlen-lasx.S @@ -0,0 +1,63 @@ +/* Optimized strlen implementation using loongarch LASX SIMD instructions. + Copyright (C) 2023 Free Software Foundation, Inc. + + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library. If not, see + . */ + +#include +#include +#include + +#if IS_IN (libc) && !defined __loongarch_soft_float + +# define STRLEN __strlen_lasx + +LEAF(STRLEN, 6) + move a1, a0 + bstrins.d a0, zero, 4, 0 + li.d t1, -1 + xvld xr0, a0, 0 + + xvmsknz.b xr0, xr0 + xvpickve.w xr1, xr0, 4 + vilvl.h vr0, vr1, vr0 + movfr2gr.s t0, fa0 # sign extend + + sra.w t0, t0, a1 + beq t0, t1, L(loop) + cto.w a0, t0 + jr ra + +L(loop): + xvld xr0, a0, 32 + addi.d a0, a0, 32 + xvsetanyeqz.b fcc0, xr0 + bceqz fcc0, L(loop) + + + xvmsknz.b xr0, xr0 + sub.d a0, a0, a1 + xvpickve.w xr1, xr0, 4 + vilvl.h vr0, vr1, vr0 + + movfr2gr.s t0, fa0 + cto.w t0, t0 + add.d a0, a0, t0 + jr ra +END(STRLEN) + +libc_hidden_builtin_def (STRLEN) +#endif diff --git a/sysdeps/loongarch/lp64/multiarch/strlen-lsx.S b/sysdeps/loongarch/lp64/multiarch/strlen-lsx.S new file mode 100644 index 0000000000..b194355e7b --- /dev/null +++ b/sysdeps/loongarch/lp64/multiarch/strlen-lsx.S @@ -0,0 +1,71 @@ +/* Optimized strlen implementation using Loongarch LSX SIMD instructions. + Copyright (C) 2023 Free Software Foundation, Inc. + + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library. If not, see + . */ + +#include +#include +#include + +#if IS_IN (libc) && !defined __loongarch_soft_float + +# define STRLEN __strlen_lsx + +LEAF(STRLEN, 6) + move a1, a0 + bstrins.d a0, zero, 4, 0 + vld vr0, a0, 0 + vld vr1, a0, 16 + + li.d t1, -1 + vmsknz.b vr0, vr0 + vmsknz.b vr1, vr1 + vilvl.h vr0, vr1, vr0 + + movfr2gr.s t0, fa0 + sra.w t0, t0, a1 + beq t0, t1, L(loop) + cto.w a0, t0 + + jr ra + nop + nop + nop + + +L(loop): + vld vr0, a0, 32 + vld vr1, a0, 48 + addi.d a0, a0, 32 + vmin.bu vr2, vr0, vr1 + + vsetanyeqz.b fcc0, vr2 + bceqz fcc0, L(loop) + vmsknz.b vr0, vr0 + vmsknz.b vr1, vr1 + + vilvl.h vr0, vr1, vr0 + sub.d a0, a0, a1 + movfr2gr.s t0, fa0 + cto.w t0, t0 + + add.d a0, a0, t0 + jr ra +END(STRLEN) + +libc_hidden_builtin_def (STRLEN) +#endif diff --git a/sysdeps/loongarch/lp64/multiarch/strlen.c b/sysdeps/loongarch/lp64/multiarch/strlen.c new file mode 100644 index 0000000000..381c2daa86 --- /dev/null +++ b/sysdeps/loongarch/lp64/multiarch/strlen.c @@ -0,0 +1,37 @@ +/* Multiple versions of strlen. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +/* Define multiple versions only for the definition in libc. */ + +#if IS_IN (libc) +# define strlen __redirect_strlen +# include +# undef strlen + +# define SYMBOL_NAME strlen +# include "ifunc-strlen.h" + +libc_ifunc_redirected (__redirect_strlen, strlen, IFUNC_SELECTOR ()); + +# ifdef SHARED +__hidden_ver1 (strlen, __GI_strlen, __redirect_strlen) + __attribute__ ((visibility ("hidden"))) __attribute_copy__ (strlen); +# endif + +#endif -- cgit 1.4.1