From 4a2c0fd44dde3f55c12517afe3658700a47b949c Mon Sep 17 00:00:00 2001 From: Adhemerval Zanella Date: Fri, 22 Nov 2013 06:38:03 -0600 Subject: PowerPC: Optimized mpn functions for PowerPC64 This patch add optimized __mpn_addmul, __mpn_addsub, __mpn_lshift, and __mpn_mul_1 implementations for PowerPC64. They are originally from GMP with adjustments for GLIBC. --- ChangeLog | 11 ++ sysdeps/powerpc/powerpc64/addmul_1.S | 208 +++++++++++++++++++++++++++++++++++ sysdeps/powerpc/powerpc64/lshift.S | 177 +++++++++++++++++++++++++++++ sysdeps/powerpc/powerpc64/mul_1.S | 135 +++++++++++++++++++++++ sysdeps/powerpc/powerpc64/submul_1.S | 21 ++++ 5 files changed, 552 insertions(+) create mode 100644 sysdeps/powerpc/powerpc64/addmul_1.S create mode 100644 sysdeps/powerpc/powerpc64/lshift.S create mode 100644 sysdeps/powerpc/powerpc64/mul_1.S create mode 100644 sysdeps/powerpc/powerpc64/submul_1.S diff --git a/ChangeLog b/ChangeLog index c03e0567a8..38fdcd832b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,14 @@ +2013-12-06 Adhemerval Zanella + + * sysdeps/powerpc/powerpc64/addmul_1.S: New file: optimized + __mpn_addmul_1 for PowerPC64. + * sysdeps/powerpc/powerpc64/submul_1.S: New file: optimized + __mpn_submul_1 for PowerPC64. + * sysdeps/powerpc/powerpc64/lshift.S: New file: optimized __mpn_lshift + for PowerPC64. + * sysdeps/powerpc/powerpc64/mul_1.S: New file: optimized __mpn_mul_1 + for PowerPC64. + 2013-12-06 Fernando J. V. da Silva [BZ #15089] diff --git a/sysdeps/powerpc/powerpc64/addmul_1.S b/sysdeps/powerpc/powerpc64/addmul_1.S new file mode 100644 index 0000000000..f256506304 --- /dev/null +++ b/sysdeps/powerpc/powerpc64/addmul_1.S @@ -0,0 +1,208 @@ +/* PowerPC64 __mpn_addmul_1 -- Multiply a limb vector with a limb and add + the result to a second limb vector. + Copyright (C) 1999-2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include + +#ifdef USE_AS_SUBMUL +# define FUNC __mpn_submul_1 +# define ADDSUBC subfe +# define ADDSUB subfc +#else +# define FUNC __mpn_addmul_1 +# define ADDSUBC adde +# define ADDSUB addc +#endif + +#define RP r3 +#define UP r4 +#define N r5 +#define VL r6 + +EALIGN(FUNC, 5, 0) + std r31, -8(r1) + rldicl. r0, N, 0, 62 + std r30, -16(r1) + cmpdi VL, r0, 2 + std r29, -24(r1) + addi N, N, 3 + std r28, -32(r1) + srdi N, N, 2 + std r27, -40(r1) + mtctr N + beq cr0, L(b00) + blt cr6, L(b01) + beq cr6, L(b10) + +L(b11): ld r9, 0(UP) + ld r28, 0(RP) + mulld r0, r9, VL + mulhdu r12, r9, VL + ADDSUB r0, r0, r28 + std r0, 0(RP) + addi RP, RP, 8 + ld r9, 8(UP) + ld r27, 16(UP) + addi UP, UP, 24 +#ifdef USE_AS_SUBMUL + subfe r11, r11, r11 +#endif + b L(bot) + + .align 4 +L(b00): ld r9, 0(UP) + ld r27, 8(UP) + ld r28, 0(RP) + ld r29, 8(RP) + mulld r0, r9, VL + mulhdu N, r9, VL + mulld r7, r27, VL + mulhdu r8, r27, VL + addc r7, r7, N + addze r12, r8 + ADDSUB r0, r0, r28 + std r0, 0(RP) + ADDSUBC r7, r7, r29 + std r7, 8(RP) + addi RP, RP, 16 + ld r9, 16(UP) + ld r27, 24(UP) + addi UP, UP, 32 +#ifdef USE_AS_SUBMUL + subfe r11, r11, r11 +#endif + b L(bot) + + .align 4 +L(b01): bdnz L(gt1) + ld r9, 0(UP) + ld r11, 0(RP) + mulld r0, r9, VL + mulhdu r8, r9, VL + ADDSUB r0, r0, r11 + std r0, 0(RP) +#ifdef USE_AS_SUBMUL + subfe r11, r11, r11 + addic r11, r11, 1 +#endif + addze RP, r8 + blr + +L(gt1): ld r9, 0(UP) + ld r27, 8(UP) + mulld r0, r9, VL + mulhdu N, r9, VL + mulld r7, r27, VL + mulhdu r8, r27, VL + ld r9, 16(UP) + ld r28, 0(RP) + ld r29, 8(RP) + ld r30, 16(RP) + mulld r11, r9, VL + mulhdu r10, r9, VL + addc r7, r7, N + adde r11, r11, r8 + addze r12, r10 + ADDSUB r0, r0, r28 + std r0, 0(RP) + ADDSUBC r7, r7, r29 + std r7, 8(RP) + ADDSUBC r11, r11, r30 + std r11, 16(RP) + addi RP, RP, 24 + ld r9, 24(UP) + ld r27, 32(UP) + addi UP, UP, 40 +#ifdef USE_AS_SUBMUL + subfe r11, r11, r11 +#endif + b L(bot) + +L(b10): addic r0, r0, r0 + li r12, 0 + ld r9, 0(UP) + ld r27, 8(UP) + bdz L(end) + addi UP, UP, 16 + + .align 4 +L(top): mulld r0, r9, VL + mulhdu N, r9, VL + mulld r7, r27, VL + mulhdu r8, r27, VL + ld r9, 0(UP) + ld r28, 0(RP) + ld r27, 8(UP) + ld r29, 8(RP) + adde r0, r0, r12 + adde r7, r7, N + mulld N, r9, VL + mulhdu r10, r9, VL + mulld r11, r27, VL + mulhdu r12, r27, VL + ld r9, 16(UP) + ld r30, 16(RP) + ld r27, 24(UP) + ld r31, 24(RP) + adde N, N, r8 + adde r11, r11, r10 + addze r12, r12 + ADDSUB r0, r0, r28 + std r0, 0(RP) + ADDSUBC r7, r7, r29 + std r7, 8(RP) + ADDSUBC N, N, r30 + std N, 16(RP) + ADDSUBC r11, r11, r31 + std r11, 24(RP) + addi UP, UP, 32 +#ifdef USE_AS_SUBMUL + subfe r11, r11, r11 +#endif + addi RP, RP, 32 +L(bot): +#ifdef USE_AS_SUBMUL + addic r11, r11, 1 +#endif + bdnz L(top) + +L(end): mulld r0, r9, VL + mulhdu N, r9, VL + mulld r7, r27, VL + mulhdu r8, r27, VL + ld r28, 0(RP) + ld r29, 8(RP) + adde r0, r0, r12 + adde r7, r7, N + addze r8, r8 + ADDSUB r0, r0, r28 + std r0, 0(RP) + ADDSUBC r7, r7, r29 + std r7, 8(RP) +#ifdef USE_AS_SUBMUL + subfe r11, r11, r11 + addic r11, r11, 1 +#endif + addze RP, r8 + ld r31, -8(r1) + ld r30, -16(r1) + ld r29, -24(r1) + ld r28, -32(r1) + ld r27, -40(r1) + blr +END(FUNC) diff --git a/sysdeps/powerpc/powerpc64/lshift.S b/sysdeps/powerpc/powerpc64/lshift.S new file mode 100644 index 0000000000..a997451c45 --- /dev/null +++ b/sysdeps/powerpc/powerpc64/lshift.S @@ -0,0 +1,177 @@ +/* PowerPC64 mpn_lshift -- rp[] = up[] << cnt + Copyright (C) 2003-2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include + +#define RP r3 +#define UP r4 +#define N r5 +#define CNT r6 + +#define TNC r0 +#define U0 r30 +#define U1 r31 +#define RETVAL r5 + +EALIGN(__mpn_lshift, 5, 0) + std U1, -8(r1) + std U0, -16(r1) + subfic TNC, CNT, 64 + sldi r7, N, RP + add UP, UP, r7 + add RP, RP, r7 + rldicl. U0, N, 0, 62 + cmpdi CNT, U0, 2 + addi U1, N, RP + ld r10, -8(UP) + srd RETVAL, r10, TNC + + srdi U1, U1, 2 + mtctr U1 + beq cr0, L(b00) + blt cr6, L(b01) + ld r11, -16(UP) + beq cr6, L(b10) + + .align 4 +L(b11): sld r8, r10, CNT + srd r9, r11, TNC + ld U1, -24(UP) + addi UP, UP, -24 + sld r12, r11, CNT + srd r7, U1, TNC + addi RP, RP, 16 + bdnz L(gt3) + + or r11, r8, r9 + sld r8, U1, CNT + b L(cj3) + + .align 4 +L(gt3): ld U0, -8(UP) + or r11, r8, r9 + sld r8, U1, CNT + srd r9, U0, TNC + ld U1, -16(UP) + or r10, r12, r7 + b L(L11) + + .align 5 +L(b10): sld r12, r10, CNT + addi RP, RP, 24 + srd r7, r11, TNC + bdnz L(gt2) + + sld r8, r11, CNT + or r10, r12, r7 + b L(cj2) + +L(gt2): ld U0, -24(UP) + sld r8, r11, CNT + srd r9, U0, TNC + ld U1, -32(UP) + or r10, r12, r7 + sld r12, U0, CNT + srd r7, U1, 0 + ld U0, -40(UP) + or r11, r8, r9 + addi UP, UP, -16 + b L(L10) + + .align 4 +L(b00): ld U1, -16(UP) + sld r12, r10, CNT + srd r7, U1, TNC + ld U0, -24(UP) + sld r8, U1, CNT + srd r9, U0, TNC + ld U1, -32(UP) + or r10, r12, r7 + sld r12, U0, CNT + srd r7, U1, TNC + addi RP, RP, r8 + bdz L(cj4) + +L(gt4): addi UP, UP, -32 + ld U0, -8(UP) + or r11, r8, r9 + b L(L00) + + .align 4 +L(b01): bdnz L(gt1) + sld r8, r10, CNT + std r8, -8(RP) + b L(ret) + +L(gt1): ld U0, -16(UP) + sld r8, r10, CNT + srd r9, U0, TNC + ld U1, -24(UP) + sld r12, U0, CNT + srd r7, U1, TNC + ld U0, -32(UP) + or r11, r8, r9 + sld r8, U1, CNT + srd r9, U0, TNC + ld U1, -40(UP) + addi UP, UP, -40 + or r10, r12, r7 + bdz L(end) + + .align 5 +L(top): sld r12, U0, CNT + srd r7, U1, TNC + ld U0, -8(UP) + std r11, -8(RP) + or r11, r8, r9 +L(L00): sld r8, U1, CNT + srd r9, U0, TNC + ld U1, -16(UP) + std r10, -16(RP) + or r10, r12, r7 +L(L11): sld r12, U0, CNT + srd r7, U1, TNC + ld U0, -24(UP) + std r11, -24(RP) + or r11, r8, r9 +L(L10): sld r8, U1, CNT + srd r9, U0, TNC + ld U1, -32(UP) + addi UP, UP, -32 + std r10, -32(RP) + addi RP, RP, -32 + or r10, r12, r7 + bdnz L(top) + + .align 5 +L(end): sld r12, U0, CNT + srd r7, U1, TNC + std r11, -8(RP) +L(cj4): or r11, r8, r9 + sld r8, U1, CNT + std r10, -16(RP) +L(cj3): or r10, r12, r7 + std r11, -24(RP) +L(cj2): std r10, -32(RP) + std r8, -40(RP) + +L(ret): ld U1, -8(r1) + ld U0, -16(r1) + mr RP, RETVAL + blr +END(__mpn_lshift) diff --git a/sysdeps/powerpc/powerpc64/mul_1.S b/sysdeps/powerpc/powerpc64/mul_1.S new file mode 100644 index 0000000000..68a1646462 --- /dev/null +++ b/sysdeps/powerpc/powerpc64/mul_1.S @@ -0,0 +1,135 @@ +/* PowerPC64 __mpn_mul_1 -- Multiply a limb vector with a limb and store + the result in a second limb vector. + Copyright (C) 1999-2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include + +#define RP r3 +#define UP r4 +#define N r5 +#define VL r6 + +EALIGN(__mpn_mul_1, 5, 0) + std r27, -40(r1) + std r26, -48(r1) + li r12, 0 + ld r26, 0(UP) + + rldicl. r0, N, 0, 62 + cmpdi VL, r0, 2 + addic N, N, RP + srdi N, N, 2 + mtctr N + beq cr0, L(b00) + blt cr6, L(b01) + beq cr6, L(b10) + +L(b11): mr cr7, r12 + mulld cr0, r26, VL + mulhdu r12, r26, VL + addi UP, UP, 8 + addc r0, r0, r7 + std r0, 0(RP) + addi RP, RP, 8 + b L(fic) + +L(b00): ld r27, r8(UP) + addi UP, UP, 16 + mulld r0, r26, VL + mulhdu N, r26, VL + mulld r7, r27, VL + mulhdu r8, r27, VL + addc r0, r0, r12 + adde r7, r7, N + addze r12, r8 + std r0, 0(RP) + std r7, 8(RP) + addi RP, RP, 16 + b L(fic) + + nop +L(b01): bdnz L(gt1) + mulld r0, r26, VL + mulhdu r8, r26, VL + addc r0, r0, r12 + std r0, 0(RP) + b L(ret) +L(gt1): ld r27, 8(UP) + nop + mulld r0, r26, VL + mulhdu N, r26, VL + ld r26, 16(UP) + mulld r7, r27, VL + mulhdu r8, r27, VL + mulld r9, r26, VL + mulhdu r10, r26, VL + addc r0, r0, r12 + adde r7, r7, N + adde r9, r9, r8 + addze r12, r10 + std r0, 0(RP) + std r7, 8(RP) + std r9, 16(RP) + addi UP, UP, 24 + addi RP, RP, 24 + b L(fic) + + nop +L(fic): ld r26, 0(UP) +L(b10): ld r27, 8(UP) + addi UP, UP, 16 + bdz L(end) + +L(top): mulld r0, r26, VL + mulhdu N, r26, VL + mulld r7, r27, VL + mulhdu r8, r27, VL + ld r26, 0(UP) + ld r27, 8(UP) + adde r0, r0, r12 + adde r7, r7, N + mulld r9, r26, VL + mulhdu r10, r26, VL + mulld r11, r27, VL + mulhdu r12, r27, VL + ld r26, 16(UP) + ld r27, 24(UP) + std r0, 0(RP) + adde r9, r9, r8 + std r7, 8(RP) + adde r11, r11, r10 + std r9, 16(RP) + addi UP, UP, 32 + std r11, 24(RP) + + addi RP, RP, 32 + bdnz L(top) + +L(end): mulld r0, r26, VL + mulhdu N, r26, VL + mulld r7, r27, VL + mulhdu r8, r27, VL + adde r0, r0, r12 + adde r7, r7, N + std r0, 0(RP) + std r7, 8(RP) +L(ret): addze RP, r8 + ld r27, -40(r1) + ld r26, -48(r1) + blr +END(__mpn_mul_1) diff --git a/sysdeps/powerpc/powerpc64/submul_1.S b/sysdeps/powerpc/powerpc64/submul_1.S new file mode 100644 index 0000000000..145b1d403c --- /dev/null +++ b/sysdeps/powerpc/powerpc64/submul_1.S @@ -0,0 +1,21 @@ +/* PowerPC64 __mpn_addmul_1 -- Multiply a limb vector with a limb and subtract + the result to a second limb vector. + Copyright (C) 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#define USE_AS_SUBMUL +#include "addmul_1.S" -- cgit 1.4.1