Optimizations from GMP.

	* sysdeps/x86_64/add_n.S: New file.
	* sysdeps/x86_64/addmul_1.S: New file.
	* sysdeps/x86_64/lshift.S: New file.
	* sysdeps/x86_64/mul_1.S: New file.
	* sysdeps/x86_64/rshift.S: New file.
	* sysdeps/x86_64/sub_n.S: New file.
	* sysdeps/x86_64/submul_1.S: New file.

7 files changed, 309 insertions, 0 deletions

diff --git a/ChangeLog b/ChangeLog
index 977136af00..5050a629c7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2009-04-14  Ulrich Drepper  <drepper@redhat.com>
+
+	Optimizations from GMP.
+	* sysdeps/x86_64/add_n.S: New file.
+	* sysdeps/x86_64/addmul_1.S: New file.
+	* sysdeps/x86_64/lshift.S: New file.
+	* sysdeps/x86_64/mul_1.S: New file.
+	* sysdeps/x86_64/rshift.S: New file.
+	* sysdeps/x86_64/sub_n.S: New file.
+	* sysdeps/x86_64/submul_1.S: New file.
+
 2009-04-14  Jakub Jelinek  <jakub@redhat.com>
 
 	* sysdeps/powerpc/powerpc32/elf/start.S: Ensure .data is non-empty.
diff --git a/sysdeps/x86_64/addmul_1.S b/sysdeps/x86_64/addmul_1.S
new file mode 100644
index 0000000000..bdb5226a33
--- /dev/null
+++ b/sysdeps/x86_64/addmul_1.S
@@ -0,0 +1,46 @@
+/* AMD64 __mpn_addmul_1 -- Multiply a limb vector with a limb and add
+   the result to a second limb vector.
+   Copyright (C) 2004 Free Software Foundation, Inc.
+   This file is part of the GNU MP Library.
+
+   The GNU MP Library is free software; you can redistribute it and/or modify
+   it under the terms of the GNU Lesser General Public License as published by
+   the Free Software Foundation; either version 2.1 of the License, or (at your
+   option) any later version.
+
+   The GNU MP Library is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+   License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with the GNU MP Library; see the file COPYING.LIB.  If not, write to
+   the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
+   MA 02111-1307, USA. */
+
+#include "sysdep.h"
+#include "asm-syntax.h"
+
+	.text
+ENTRY (__mpn_addmul_1)
+	movq	%rdx, %r11
+	leaq	(%rsi,%rdx,8), %rsi
+	leaq	(%rdi,%rdx,8), %rdi
+	negq	%r11
+	xorl	%r8d, %r8d
+	xorl	%r10d, %r10d
+	.p2align 2
+L(loop):
+	movq	(%rsi,%r11,8), %rax
+	mulq	%rcx
+	addq	(%rdi,%r11,8), %rax
+	adcq	%r10, %rdx
+	addq	%r8, %rax
+	movq	%r10, %r8
+	movq	%rax, (%rdi,%r11,8)
+	adcq	%rdx, %r8
+	incq	%r11
+	jne	L(loop)
+	movq	%r8, %rax
+	ret
+END (__mpn_addmul_1)
diff --git a/sysdeps/x86_64/lshift.S b/sysdeps/x86_64/lshift.S
new file mode 100644
index 0000000000..5ac66f0a36
--- /dev/null
+++ b/sysdeps/x86_64/lshift.S
@@ -0,0 +1,60 @@
+/* AMD64 __mpn_lshift --
+   Copyright 2004, 2006 Free Software Foundation, Inc.
+   This file is part of the GNU MP Library.
+
+   The GNU MP Library is free software; you can redistribute it and/or modify
+   it under the terms of the GNU Lesser General Public License as published by
+   the Free Software Foundation; either version 2.1 of the License, or (at your
+   option) any later version.
+
+   The GNU MP Library is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+   License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with the GNU MP Library; see the file COPYING.LIB.  If not, write to
+   the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
+   MA 02111-1307, USA. */
+
+#include "sysdep.h"
+#include "asm-syntax.h"
+
+
+	.text
+ENTRY (__mpn_lshift)
+	movq	-8(%rsi,%rdx,8), %mm7
+	movd	%ecx, %mm1
+	movl	$64, %eax
+	subl	%ecx, %eax
+	movd	%eax, %mm0
+	movq	%mm7, %mm3
+	psrlq	%mm0, %mm7
+	movd	%mm7, %rax
+	subq	$2, %rdx
+	jl	L(endo)
+	.p2align 2
+L(loop):
+	movq	(%rsi,%rdx,8), %mm6
+	movq	%mm6, %mm2
+	psrlq	%mm0, %mm6
+	psllq	%mm1, %mm3
+	por	%mm6, %mm3
+	movq	%mm3, 8(%rdi,%rdx,8)
+	je	L(ende)
+	movq	-8(%rsi,%rdx,8), %mm7
+	movq	%mm7, %mm3
+	psrlq	%mm0, %mm7
+	psllq	%mm1, %mm2
+	por	%mm7, %mm2
+	movq	%mm2, (%rdi,%rdx,8)
+	subq	$2, %rdx
+	jge	L(loop)
+L(endo):
+	movq	%mm3, %mm2
+L(ende):
+	psllq	%mm1, %mm2
+	movq	%mm2, (%rdi)
+	emms
+	ret
+END (__mpn_lshift)
diff --git a/sysdeps/x86_64/mul_1.S b/sysdeps/x86_64/mul_1.S
new file mode 100644
index 0000000000..978916b72c
--- /dev/null
+++ b/sysdeps/x86_64/mul_1.S
@@ -0,0 +1,42 @@
+/* AMD64 __mpn_mul_1 -- Multiply a limb vector with a limb and store
+   the result in a second limb vector.
+   Copyright (C) 2004 Free Software Foundation, Inc.
+   This file is part of the GNU MP Library.
+
+   The GNU MP Library is free software; you can redistribute it and/or modify
+   it under the terms of the GNU Lesser General Public License as published by
+   the Free Software Foundation; either version 2.1 of the License, or (at your
+   option) any later version.
+
+   The GNU MP Library is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+   License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with the GNU MP Library; see the file COPYING.LIB.  If not, write to
+   the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
+   MA 02111-1307, USA. */
+
+#include <sysdep.h>
+#include "asm-syntax.h"
+
+	.text
+ENTRY (__mpn_mul_1)
+	movq	%rdx, %r11
+	leaq	(%rsi,%rdx,8), %rsi
+	leaq	(%rdi,%rdx,8), %rdi
+	negq	%r11
+	xorl	%r8d, %r8d
+L(loop):
+	movq	(%rsi,%r11,8), %rax
+	mulq	%rcx
+	addq	%r8, %rax
+	movl	$0, %r8d
+	adcq	%rdx, %r8
+	movq	%rax, (%rdi,%r11,8)
+	incq	%r11
+	jne	L(loop)
+	movq	%r8, %rax
+	ret
+END (__mpn_mul_1)
diff --git a/sysdeps/x86_64/rshift.S b/sysdeps/x86_64/rshift.S
new file mode 100644
index 0000000000..ee0c8aa15c
--- /dev/null
+++ b/sysdeps/x86_64/rshift.S
@@ -0,0 +1,62 @@
+/* AMD64 __mpn_rshift --
+   Copyright (C) 2004, 2006 Free Software Foundation, Inc.
+   This file is part of the GNU MP Library.
+
+   The GNU MP Library is free software; you can redistribute it and/or modify
+   it under the terms of the GNU Lesser General Public License as published by
+   the Free Software Foundation; either version 2.1 of the License, or (at your
+   option) any later version.
+
+   The GNU MP Library is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+   License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with the GNU MP Library; see the file COPYING.LIB.  If not, write to
+   the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
+   MA 02111-1307, USA. */
+
+#include "sysdep.h"
+#include "asm-syntax.h"
+
+	.text
+ENTRY (__mpn_rshift)
+	movq	(%rsi), %mm7
+	movd	%ecx, %mm1
+	movl	$64, %eax
+	subl	%ecx, %eax
+	movd	%eax, %mm0
+	movq	%mm7, %mm3
+	psllq	%mm0, %mm7
+	movd	%mm7, %rax
+	leaq	(%rsi,%rdx,8), %rsi
+	leaq	(%rdi,%rdx,8), %rdi
+	negq	%rdx
+	addq	$2, %rdx
+	jg	L(endo)
+	.p2align 2
+L(loop):
+	movq	-8(%rsi,%rdx,8), %mm6
+	movq	%mm6, %mm2
+	psllq	%mm0, %mm6
+	psrlq	%mm1, %mm3
+	por	%mm6, %mm3
+	movq	%mm3, -16(%rdi,%rdx,8)
+	je	L(ende)
+	movq	(%rsi,%rdx,8), %mm7
+	movq	%mm7, %mm3
+	psllq	%mm0, %mm7
+	psrlq	%mm1, %mm2
+	por	%mm7, %mm2
+	movq	%mm2, -8(%rdi,%rdx,8)
+	addq	$2, %rdx
+	jle	L(loop)
+L(endo):
+	movq	%mm3, %mm2
+L(ende):
+	psrlq	%mm1, %mm2
+	movq	%mm2, -8(%rdi)
+	emms
+	ret
+END (__mpn_rshift)
diff --git a/sysdeps/x86_64/sub_n.S b/sysdeps/x86_64/sub_n.S
new file mode 100644
index 0000000000..48e1a2e0f4
--- /dev/null
+++ b/sysdeps/x86_64/sub_n.S
@@ -0,0 +1,42 @@
+/* AMD64 __mpn_sub_n -- Add two limb vectors of the same length > 0 and store
+   sum in a third limb vector.
+   Copyright (C) 2004 Free Software Foundation, Inc.
+   This file is part of the GNU MP Library.
+
+   The GNU MP Library is free software; you can redistribute it and/or modify
+   it under the terms of the GNU Lesser General Public License as published by
+   the Free Software Foundation; either version 2.1 of the License, or (at your
+   option) any later version.
+
+   The GNU MP Library is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+   License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with the GNU MP Library; see the file COPYING.LIB.  If not, write to
+   the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
+   MA 02111-1307, USA. */
+
+#include "sysdep.h"
+#include "asm-syntax.h"
+
+	.text
+ENTRY (__mpn_sub_n)
+	leaq	(%rsi,%rcx,8), %rsi
+	leaq	(%rdi,%rcx,8), %rdi
+	leaq	(%rdx,%rcx,8), %rdx
+	negq	%rcx
+	xorl	%eax, %eax		# clear cy
+	.p2align 2
+L(loop):
+	movq	(%rsi,%rcx,8), %rax
+	movq	(%rdx,%rcx,8), %r10
+	sbbq	%r10, %rax
+	movq	%rax, (%rdi,%rcx,8)
+	incq	%rcx
+	jne	L(loop)
+	movq	%rcx, %rax		# zero %rax
+	adcq	%rax, %rax
+	ret
+END (__mpn_sub_n)
diff --git a/sysdeps/x86_64/submul_1.S b/sysdeps/x86_64/submul_1.S
new file mode 100644
index 0000000000..e94c9a7bee
--- /dev/null
+++ b/sysdeps/x86_64/submul_1.S
@@ -0,0 +1,46 @@
+/* AMD64 __mpn_submul_1 -- Multiply a limb vector with a limb and subtract
+   the result from a second limb vector.
+   Copyright (C) 2004 Free Software Foundation, Inc.
+   This file is part of the GNU MP Library.
+
+   The GNU MP Library is free software; you can redistribute it and/or modify
+   it under the terms of the GNU Lesser General Public License as published by
+   the Free Software Foundation; either version 2.1 of the License, or (at your
+   option) any later version.
+
+   The GNU MP Library is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+   License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with the GNU MP Library; see the file COPYING.LIB.  If not, write to
+   the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
+   MA 02111-1307, USA. */
+
+#include "sysdep.h"
+#include "asm-syntax.h"
+
+	.text
+ENTRY (__mpn_submul_1)
+	movq	%rdx, %r11
+	leaq	(%rsi,%r11,8), %rsi
+	leaq	(%rdi,%r11,8), %rdi
+	negq	%r11
+	xorl	%r8d, %r8d
+	.p2align 3
+L(loop):
+	movq	(%rsi,%r11,8), %rax
+	movq	(%rdi,%r11,8), %r10
+	mulq	%rcx
+	subq	%r8, %r10
+	movl	$0, %r8d
+	adcl	%r8d, %r8d
+	subq	%rax, %r10
+	adcq	%rdx, %r8
+	movq	%r10, (%rdi,%r11,8)
+	incq	%r11
+	jne	L(loop)
+	movq	%r8, %rax
+	ret
+END (__mpn_submul_1)