poly1305: add AMD64/SSE2 optimized implementation

* cipher/Makefile.am: Add 'poly1305-sse2-amd64.S'.
* cipher/poly1305-internal.h (POLY1305_USE_SSE2)
(POLY1305_SSE2_BLOCKSIZE, POLY1305_SSE2_STATESIZE)
(POLY1305_SSE2_ALIGNMENT): New.
(POLY1305_LARGEST_BLOCKSIZE, POLY1305_LARGEST_STATESIZE)
(POLY1305_STATE_ALIGNMENT): Use SSE2 versions when needed.
* cipher/poly1305-sse2-amd64.S: New.
* cipher/poly1305.c [POLY1305_USE_SSE2]
(_gcry_poly1305_amd64_sse2_init_ext)
(_gcry_poly1305_amd64_sse2_finish_ext)
(_gcry_poly1305_amd64_sse2_blocks, poly1305_amd64_sse2_ops): New.
(_gcry_polu1305_init) [POLY1305_USE_SSE2]: Use SSE2 version.
* configure.ac [host=x86_64]: Add 'poly1305-sse2-amd64.lo'.
--

Add Andrew Moon's public domain SSE2 implementation of Poly1305. Original
source is available at: https://github.com/floodyberry/poly1305-opt

Benchmarks on Intel i5-4570 (haswell):

Old:
                    |  nanosecs/byte   mebibytes/sec   cycles/byte
 POLY1305           |     0.844 ns/B    1130.2 MiB/s      2.70 c/B

New:
                    |  nanosecs/byte   mebibytes/sec   cycles/byte
 POLY1305           |     0.448 ns/B    2129.5 MiB/s      1.43 c/B

Benchmarks on Intel i5-2450M (sandy-bridge):

Old:
                    |  nanosecs/byte   mebibytes/sec   cycles/byte
 POLY1305           |      1.25 ns/B     763.0 MiB/s      3.12 c/B

New:
                    |  nanosecs/byte   mebibytes/sec   cycles/byte
 POLY1305           |     0.605 ns/B    1575.9 MiB/s      1.51 c/B

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>

5 files changed, 1091 insertions, 3 deletions

diff --git a/cipher/Makefile.am b/cipher/Makefile.am
index 44686470..a32ae892 100644
--- a/cipher/Makefile.am
+++ b/cipher/Makefile.am
@@ -72,6 +72,7 @@ gost28147.c gost.h \
 gostr3411-94.c \
 md4.c \
 md5.c \
+poly1305-sse2-amd64.S \
 rijndael.c rijndael-tables.h rijndael-amd64.S rijndael-arm.S \
 rmd160.c \
 rsa.c \
diff --git a/cipher/poly1305-internal.h b/cipher/poly1305-internal.h
index d2c6b5cd..fa3fe75e 100644
--- a/cipher/poly1305-internal.h
+++ b/cipher/poly1305-internal.h
@@ -44,15 +44,37 @@
 #define POLY1305_REF_ALIGNMENT sizeof(void *)
 
 
+/* POLY1305_USE_SSE2 indicates whether to compile with AMD64 SSE2 code. */
+#undef POLY1305_USE_SSE2
+#if defined(__x86_64__) && defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS)
+# define POLY1305_USE_SSE2 1
+# define POLY1305_SSE2_BLOCKSIZE 32
+# define POLY1305_SSE2_STATESIZE 248
+# define POLY1305_SSE2_ALIGNMENT 16
+#endif
+
+
 /* Largest block-size used in any implementation (optimized implementations
  * might use block-size multiple of 16). */
-#define POLY1305_LARGEST_BLOCKSIZE POLY1305_REF_BLOCKSIZE
+#ifdef POLY1305_USE_SSE2
+# define POLY1305_LARGEST_BLOCKSIZE POLY1305_SSE2_BLOCKSIZE
+#else
+# define POLY1305_LARGEST_BLOCKSIZE POLY1305_REF_BLOCKSIZE
+#endif
 
 /* Largest state-size used in any implementation. */
-#define POLY1305_LARGEST_STATESIZE POLY1305_REF_STATESIZE
+#ifdef POLY1305_USE_SSE2
+# define POLY1305_LARGEST_STATESIZE POLY1305_SSE2_STATESIZE
+#else
+# define POLY1305_LARGEST_STATESIZE POLY1305_REF_STATESIZE
+#endif
 
 /* Minimum alignment for state pointer passed to implementations. */
-#define POLY1305_STATE_ALIGNMENT POLY1305_REF_ALIGNMENT
+#ifdef POLY1305_USE_SSE2
+# define POLY1305_STATE_ALIGNMENT POLY1305_SSE2_ALIGNMENT
+#else
+# define POLY1305_STATE_ALIGNMENT POLY1305_REF_ALIGNMENT
+#endif
 
 
 typedef struct poly1305_key_s
diff --git a/cipher/poly1305-sse2-amd64.S b/cipher/poly1305-sse2-amd64.S
new file mode 100644
index 00000000..106b1197
--- /dev/null
+++ b/cipher/poly1305-sse2-amd64.S
@@ -0,0 +1,1035 @@
+/* poly1305-sse2-amd64.S  -  AMD64/SSE2 implementation of Poly1305
+ *
+ * Copyright (C) 2014 Jussi Kivilinna <jussi.kivilinna@iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * Based on public domain implementation by Andrew Moon at
+ *  https://github.com/floodyberry/poly1305-opt
+ */
+
+#include <config.h>
+
+#if defined(__x86_64__) && defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS)
+
+.text
+
+
+.align 8
+.globl _gcry_poly1305_amd64_sse2_init_ext
+.type  _gcry_poly1305_amd64_sse2_init_ext,@function;
+_gcry_poly1305_amd64_sse2_init_ext:
+.Lpoly1305_init_ext_x86_local:
+	xor %edx, %edx
+	pushq %r12
+	pushq %r13
+	pushq %r14
+	movq %rdx, %r10
+	movq $-1, %rcx
+	testq %r10, %r10
+	pxor %xmm0, %xmm0
+	movq $0xfffffc0ffff, %r9
+	movdqa %xmm0, (%rdi)
+	cmove %rcx, %r10
+	movdqa %xmm0, 16(%rdi)
+	movq $0xffc0fffffff, %rcx
+	movdqa %xmm0, 32(%rdi)
+	movdqa %xmm0, 48(%rdi)
+	movdqa %xmm0, 64(%rdi)
+	movq 8(%rsi), %r11
+	movq %r11, %r8
+	movq (%rsi), %r12
+	andq %r12, %rcx
+	shrq $44, %r12
+	shlq $20, %r8
+	shrq $24, %r11
+	orq %r8, %r12
+	movq $0xffffffc0f, %r8
+	andq %r9, %r12
+	andq %r8, %r11
+	movl %ecx, %r8d
+	andl $67108863, %r8d
+	movq %rcx, %r9
+	movl %r8d, 84(%rdi)
+	movq %r12, %r8
+	shrq $26, %r9
+	shlq $18, %r8
+	orq %r8, %r9
+	movq %r12, %r8
+	shrq $8, %r8
+	andl $67108863, %r9d
+	andl $67108863, %r8d
+	movl %r9d, 92(%rdi)
+	movq %r12, %r9
+	movl %r8d, 100(%rdi)
+	movq %r11, %r8
+	shrq $34, %r9
+	shlq $10, %r8
+	orq %r8, %r9
+	movq %r11, %r8
+	shrq $16, %r8
+	andl $67108863, %r9d
+	movl %r9d, 108(%rdi)
+	cmpq $16, %r10
+	movl %r8d, 116(%rdi)
+	movl 16(%rsi), %r8d
+	movl %r8d, 124(%rdi)
+	movl 20(%rsi), %r8d
+	movl %r8d, 132(%rdi)
+	movl 24(%rsi), %r8d
+	movl %r8d, 140(%rdi)
+	movl 28(%rsi), %esi
+	movl %esi, 148(%rdi)
+	jbe .Lpoly1305_init_ext_sse2_done
+	lea (%r11,%r11,4), %r14
+	shlq $2, %r14
+	lea (%r12,%r12), %rax
+	mulq %r14
+	movq %rax, %r13
+	movq %rcx, %rax
+	movq %rdx, %r8
+	mulq %rcx
+	addq %rax, %r13
+	lea (%rcx,%rcx), %rax
+	movq %r13, %r9
+	adcq %rdx, %r8
+	mulq %r12
+	shlq $20, %r8
+	movq %rax, %rsi
+	shrq $44, %r9
+	movq %r11, %rax
+	orq %r9, %r8
+	movq %rdx, %r9
+	mulq %r14
+	addq %rax, %rsi
+	movq %rcx, %rax
+	adcq %rdx, %r9
+	addq %r11, %r11
+	mulq %r11
+	addq %rsi, %r8
+	movq %rax, %r11
+	movq %r12, %rax
+	movq %rdx, %rcx
+	adcq $0, %r9
+	mulq %r12
+	addq %rax, %r11
+	movq %r8, %rsi
+	adcq %rdx, %rcx
+	shlq $20, %r9
+	shrq $44, %rsi
+	orq %rsi, %r9
+	movq $0xfffffffffff, %rsi
+	addq %r11, %r9
+	movq %r9, %r12
+	adcq $0, %rcx
+	andq %rsi, %r13
+	shlq $22, %rcx
+	andq %rsi, %r8
+	shrq $42, %r12
+	orq %r12, %rcx
+	movq %rsi, %r12
+	lea (%rcx,%rcx,4), %rcx
+	addq %rcx, %r13
+	movq %rsi, %rcx
+	andq %r13, %rcx
+	shrq $44, %r13
+	movq %rcx, %r14
+	addq %r13, %r8
+	movq $0x3ffffffffff, %r13
+	andq %r8, %r12
+	andq %r13, %r9
+	shrq $44, %r8
+	movq %r12, %r11
+	addq %r8, %r9
+	movq %r12, %rax
+	movq %r9, %r13
+	movl %ecx, %r8d
+	shrq $26, %r14
+	andl $67108863, %r8d
+	shlq $18, %r11
+	shrq $34, %rax
+	orq %r11, %r14
+	shlq $10, %r13
+	movq %r12, %r11
+	orq %r13, %rax
+	movq %r9, %r13
+	shrq $8, %r11
+	shrq $16, %r13
+	andl $67108863, %r14d
+	andl $67108863, %r11d
+	andl $67108863, %eax
+	movl %r8d, 88(%rdi)
+	cmpq $64, %r10
+	movl %r8d, 80(%rdi)
+	movl %r14d, 104(%rdi)
+	movl %r14d, 96(%rdi)
+	movl %r11d, 120(%rdi)
+	movl %r11d, 112(%rdi)
+	movl %eax, 136(%rdi)
+	movl %eax, 128(%rdi)
+	movl %r13d, 152(%rdi)
+	movl %r13d, 144(%rdi)
+	jbe .Lpoly1305_init_ext_sse2_done
+	lea (%r9,%r9,4), %r14
+	shlq $2, %r14
+	lea (%r12,%r12), %rax
+	mulq %r14
+	movq %rax, %r8
+	movq %rcx, %rax
+	movq %rdx, %r10
+	mulq %rcx
+	addq %rax, %r8
+	lea (%rcx,%rcx), %rax
+	movq %r8, %r11
+	adcq %rdx, %r10
+	andq %rsi, %r8
+	mulq %r12
+	shlq $20, %r10
+	movq %rax, %r13
+	shrq $44, %r11
+	movq %r9, %rax
+	orq %r11, %r10
+	movq %rdx, %r11
+	mulq %r14
+	addq %rax, %r13
+	movq %rcx, %rax
+	adcq %rdx, %r11
+	addq %r9, %r9
+	mulq %r9
+	addq %r13, %r10
+	movq %rax, %r9
+	movq %r12, %rax
+	movq %rdx, %rcx
+	adcq $0, %r11
+	mulq %r12
+	addq %rax, %r9
+	movq %r10, %r13
+	adcq %rdx, %rcx
+	andq %rsi, %r10
+	shlq $20, %r11
+	shrq $44, %r13
+	orq %r13, %r11
+	addq %r9, %r11
+	movq %rsi, %r9
+	movq %r11, %r12
+	adcq $0, %rcx
+	shlq $22, %rcx
+	shrq $42, %r12
+	orq %r12, %rcx
+	lea (%rcx,%rcx,4), %rcx
+	addq %rcx, %r8
+	andq %r8, %r9
+	shrq $44, %r8
+	movl %r9d, %eax
+	addq %r8, %r10
+	movq $0x3ffffffffff, %r8
+	andq %r10, %rsi
+	andq %r8, %r11
+	shrq $44, %r10
+	movq %rsi, %r8
+	addq %r10, %r11
+	andl $67108863, %eax
+	shrq $26, %r9
+	movq %r11, %r10
+	shlq $18, %r8
+	shlq $10, %r10
+	orq %r8, %r9
+	movq %rsi, %r8
+	shrq $34, %rsi
+	andl $67108863, %r9d
+	shrq $8, %r8
+	orq %r10, %rsi
+	shrq $16, %r11
+	andl $67108863, %r8d
+	andl $67108863, %esi
+	movl %eax, 168(%rdi)
+	movl %eax, 160(%rdi)
+	movl %r9d, 184(%rdi)
+	movl %r9d, 176(%rdi)
+	movl %r8d, 200(%rdi)
+	movl %r8d, 192(%rdi)
+	movl %esi, 216(%rdi)
+	movl %esi, 208(%rdi)
+	movl %r11d, 232(%rdi)
+	movl %r11d, 224(%rdi)
+.Lpoly1305_init_ext_sse2_done:
+	movq $0, 240(%rdi)
+	popq %r14
+	popq %r13
+	popq %r12
+	ret
+.size _gcry_poly1305_amd64_sse2_init_ext,.-_gcry_poly1305_amd64_sse2_init_ext;
+
+
+.align 8
+.globl _gcry_poly1305_amd64_sse2_finish_ext
+.type  _gcry_poly1305_amd64_sse2_finish_ext,@function;
+_gcry_poly1305_amd64_sse2_finish_ext:
+.Lpoly1305_finish_ext_x86_local:
+	pushq %rbp
+	movq %rsp, %rbp
+	subq $64, %rsp
+	andq $~63, %rsp
+	movq %rdx, 32(%rsp)
+	movq %rcx, 40(%rsp)
+	andq %rdx, %rdx
+	jz .Lpoly1305_finish_x86_no_leftover
+	pxor %xmm0, %xmm0
+	movdqa %xmm0, 0+0(%rsp)
+	movdqa %xmm0, 16+0(%rsp)
+	leaq 0(%rsp), %r8
+	testq $16, %rdx
+	jz .Lpoly1305_finish_x86_skip16
+	movdqu 0(%rsi), %xmm0
+	movdqa %xmm0, 0(%r8)
+	addq $16, %rsi
+	addq $16, %r8
+.Lpoly1305_finish_x86_skip16:
+	testq $8, %rdx
+	jz .Lpoly1305_finish_x86_skip8
+	movq 0(%rsi), %rax
+	movq %rax, 0(%r8)
+	addq $8, %rsi
+	addq $8, %r8
+.Lpoly1305_finish_x86_skip8:
+	testq $4, %rdx
+	jz .Lpoly1305_finish_x86_skip4
+	movl 0(%rsi), %eax
+	movl %eax, 0(%r8)
+	addq $4, %rsi
+	addq $4, %r8
+.Lpoly1305_finish_x86_skip4:
+	testq $2, %rdx
+	jz .Lpoly1305_finish_x86_skip2
+	movw 0(%rsi), %ax
+	movw %ax, 0(%r8)
+	addq $2, %rsi
+	addq $2, %r8
+.Lpoly1305_finish_x86_skip2:
+	testq $1, %rdx
+	jz .Lpoly1305_finish_x86_skip1
+	movb 0(%rsi), %al
+	movb %al, 0(%r8)
+	addq $1, %r8
+.Lpoly1305_finish_x86_skip1:
+	cmpq $16, %rdx
+	je .Lpoly1305_finish_x86_is16
+	movb $1, 0(%r8)
+.Lpoly1305_finish_x86_is16:
+	movq $4, %rax
+	jae .Lpoly1305_finish_x86_16andover
+	movq $8, %rax
+.Lpoly1305_finish_x86_16andover:
+	orq %rax, 240(%rdi)
+	leaq 0(%rsp), %rsi
+	movq $32, %rdx
+	callq .Lpoly1305_blocks_x86_local
+.Lpoly1305_finish_x86_no_leftover:
+	testq $1, 240(%rdi)
+	jz .Lpoly1305_finish_x86_not_started
+	movq 32(%rsp), %rdx
+	andq %rdx, %rdx
+	jz .Lpoly1305_finish_x86_r2r
+	cmpq $16, %rdx
+	jg .Lpoly1305_finish_x86_r2r
+	xorl %r10d, %r10d
+	movl 84(%rdi), %eax
+	movl 92(%rdi), %ecx
+	movl 100(%rdi), %edx
+	movl 108(%rdi), %r8d
+	movl 116(%rdi), %r9d
+	movl %eax, 80(%rdi)
+	movl $1, 8+80(%rdi)
+	movl %ecx, 96(%rdi)
+	movl %r10d, 8+96(%rdi)
+	movl %edx, 112(%rdi)
+	movl %r10d, 8+112(%rdi)
+	movl %r8d, 128(%rdi)
+	movl %r10d, 8+128(%rdi)
+	movl %r9d, 144(%rdi)
+	movl %r10d, 8+144(%rdi)
+	jmp .Lpoly1305_finish_x86_combine
+.Lpoly1305_finish_x86_r2r:
+	movl 84(%rdi), %eax
+	movl 92(%rdi), %ecx
+	movl 100(%rdi), %edx
+	movl 108(%rdi), %r8d
+	movl 116(%rdi), %r9d
+	movl %eax, 8+80(%rdi)
+	movl %ecx, 8+96(%rdi)
+	movl %edx, 8+112(%rdi)
+	movl %r8d, 8+128(%rdi)
+	movl %r9d, 8+144(%rdi)
+.Lpoly1305_finish_x86_combine:
+	xorq %rsi, %rsi
+	movq $32, %rdx
+	callq .Lpoly1305_blocks_x86_local
+.Lpoly1305_finish_x86_not_started:
+	movq 0(%rdi), %r8
+	movq 8(%rdi), %r9
+	movq %r9, %r10
+	movq 16(%rdi), %r11
+	shlq $44, %r9
+	shrq $20, %r10
+	shlq $24, %r11
+	orq %r9, %r8
+	orq %r11, %r10
+	pxor %xmm0, %xmm0
+	movl 124(%rdi), %eax
+	movl 132(%rdi), %ecx
+	movl 140(%rdi), %edx
+	movl 148(%rdi), %esi
+	movq 40(%rsp), %r11
+	shlq $32, %rcx
+	shlq $32, %rsi
+	orq %rcx, %rax
+	orq %rsi, %rdx
+	addq %r8, %rax
+	adcq %r10, %rdx
+	movq %rax, 0(%r11)
+	movq %rdx, 8(%r11)
+	movq %rbp, %rax
+	subq %rsp, %rax
+	movq %rbp, %rsp
+	movdqa %xmm0, 0(%rdi)
+	movdqa %xmm0, 16(%rdi)
+	movdqa %xmm0, 32(%rdi)
+	movdqa %xmm0, 48(%rdi)
+	movdqa %xmm0, 64(%rdi)
+	movdqa %xmm0, 80(%rdi)
+	movdqa %xmm0, 96(%rdi)
+	movdqa %xmm0, 112(%rdi)
+	movdqa %xmm0, 128(%rdi)
+	movdqa %xmm0, 144(%rdi)
+	movdqa %xmm0, 160(%rdi)
+	movdqa %xmm0, 176(%rdi)
+	movdqa %xmm0, 192(%rdi)
+	movdqa %xmm0, 208(%rdi)
+	movdqa %xmm0, 224(%rdi)
+	popq %rbp
+	addq $8, %rax
+	ret
+.size _gcry_poly1305_amd64_sse2_finish_ext,.-_gcry_poly1305_amd64_sse2_finish_ext;
+
+
+.align 8
+.globl _gcry_poly1305_amd64_sse2_blocks
+.type  _gcry_poly1305_amd64_sse2_blocks,@function;
+_gcry_poly1305_amd64_sse2_blocks:
+.Lpoly1305_blocks_x86_local:
+	pushq %rbp
+	movq %rsp, %rbp
+	pushq %rbx
+	andq $-64, %rsp
+	subq $328, %rsp
+	movq 240(%rdi), %rax
+	movl $(1<<24), %r8d
+	movl $((1<<26)-1), %r9d
+	movd %r8, %xmm0
+	movd %r9, %xmm5
+	pshufd $0x44, %xmm0, %xmm0
+	pshufd $0x44, %xmm5, %xmm5
+	testb $4, %al
+	je .Lpoly1305_blocks_x86_3
+	psrldq $8, %xmm0
+.Lpoly1305_blocks_x86_3:
+	testb $8, %al
+	je .Lpoly1305_blocks_x86_4
+	pxor %xmm0, %xmm0
+.Lpoly1305_blocks_x86_4:
+	movdqa %xmm0, 168(%rsp)
+	testb $1, %al
+	jne .Lpoly1305_blocks_x86_5
+	movq 16(%rsi), %xmm0
+	movdqa %xmm5, %xmm7
+	movdqa %xmm5, %xmm10
+	movq (%rsi), %xmm6
+	orq $1, %rax
+	subq $32, %rdx
+	movq 8(%rsi), %xmm1
+	punpcklqdq %xmm0, %xmm6
+	movq 24(%rsi), %xmm0
+	pand %xmm6, %xmm7
+	movdqa %xmm6, %xmm9
+	psrlq $52, %xmm6
+	addq $32, %rsi
+	punpcklqdq %xmm0, %xmm1
+	movdqa %xmm1, %xmm0
+	psrlq $26, %xmm9
+	psllq $12, %xmm0
+	movq %rax, 240(%rdi)
+	pand %xmm5, %xmm9
+	por %xmm0, %xmm6
+	psrlq $40, %xmm1
+	pand %xmm6, %xmm10
+	por 168(%rsp), %xmm1
+	psrlq $26, %xmm6
+	pand %xmm5, %xmm6
+.Lpoly1305_blocks_x86_6:
+	movdqa 80(%rdi), %xmm13
+	cmpq $63, %rdx
+	movl $(5), %r8d
+	movd %r8, %xmm14
+	pshufd $0x44, %xmm14, %xmm14
+	movdqa 96(%rdi), %xmm15
+	movdqa %xmm13, -8(%rsp)
+	movdqa 112(%rdi), %xmm0
+	movdqa %xmm14, 136(%rsp)
+	movdqa 128(%rdi), %xmm3
+	movdqa %xmm15, 312(%rsp)
+	pmuludq %xmm14, %xmm15
+	movdqa 144(%rdi), %xmm13
+	movdqa %xmm0, 232(%rsp)
+	pmuludq %xmm14, %xmm0
+	movdqa %xmm3, 152(%rsp)
+	pmuludq %xmm14, %xmm3
+	movdqa %xmm13, 56(%rsp)
+	pmuludq %xmm14, %xmm13
+	movdqa %xmm15, 40(%rsp)
+	movdqa %xmm0, -24(%rsp)
+	movdqa %xmm3, -40(%rsp)
+	movdqa %xmm13, -56(%rsp)
+	jbe .Lpoly1305_blocks_x86_7
+	movdqa 192(%rdi), %xmm15
+	leaq 32(%rsi), %rax
+	movq %rdx, %rcx
+	movdqa 176(%rdi), %xmm14
+	movdqa %xmm15, %xmm2
+	movdqa 208(%rdi), %xmm0
+	movdqa %xmm15, 216(%rsp)
+	movdqa %xmm14, 296(%rsp)
+	movdqa 224(%rdi), %xmm3
+	pmuludq 136(%rsp), %xmm14
+	movdqa -24(%rsp), %xmm13
+	movdqa %xmm14, 8(%rsp)
+	pmuludq 136(%rsp), %xmm2
+	movdqa -40(%rsp), %xmm14
+	movdqa %xmm0, 120(%rsp)
+	pmuludq 136(%rsp), %xmm0
+	movdqa %xmm3, 24(%rsp)
+	movdqa 160(%rdi), %xmm12
+	movdqa %xmm0, %xmm8
+	movdqa -56(%rsp), %xmm15
+	movdqa %xmm13, 88(%rsp)
+	pmuludq 136(%rsp), %xmm3
+	movdqa %xmm2, 104(%rsp)
+	movdqa %xmm0, %xmm13
+	movdqa -8(%rsp), %xmm11
+	movdqa %xmm3, 280(%rsp)
+	movdqa %xmm2, %xmm3
+	movdqa %xmm0, 200(%rsp)
+	movdqa %xmm14, 184(%rsp)
+	movdqa %xmm15, 264(%rsp)
+	jmp .Lpoly1305_blocks_x86_8
+.p2align 6,,63
+.Lpoly1305_blocks_x86_13:
+	movdqa 200(%rsp), %xmm13
+	movdqa %xmm3, %xmm6
+	movdqa 200(%rsp), %xmm8
+	movdqa 104(%rsp), %xmm3
+.Lpoly1305_blocks_x86_8:
+	movdqa 8(%rsp), %xmm4
+	pmuludq %xmm6, %xmm3
+	subq $64, %rcx
+	pmuludq %xmm10, %xmm8
+	movdqa 104(%rsp), %xmm2
+	movdqa 200(%rsp), %xmm0
+	pmuludq %xmm1, %xmm4
+	movdqa 280(%rsp), %xmm15
+	pmuludq %xmm6, %xmm13
+	movdqa 280(%rsp), %xmm14
+	pmuludq %xmm1, %xmm0
+	paddq %xmm3, %xmm4
+	pmuludq %xmm1, %xmm2
+	movdqa 280(%rsp), %xmm3
+	paddq %xmm8, %xmm4
+	pmuludq %xmm9, %xmm15
+	movdqa 280(%rsp), %xmm8
+	pmuludq %xmm10, %xmm14
+	pmuludq %xmm6, %xmm8
+	paddq %xmm13, %xmm2
+	movdqa %xmm6, %xmm13
+	pmuludq %xmm1, %xmm3
+	paddq %xmm15, %xmm4
+	movdqa 296(%rsp), %xmm15
+	pmuludq %xmm12, %xmm13
+	paddq %xmm14, %xmm2
+	movdqa %xmm7, %xmm14
+	paddq %xmm8, %xmm0
+	pmuludq %xmm12, %xmm14
+	movdqa %xmm9, %xmm8
+	pmuludq 296(%rsp), %xmm6
+	pmuludq %xmm12, %xmm8
+	movdqa %xmm6, 248(%rsp)
+	pmuludq %xmm10, %xmm15
+	movq -16(%rax), %xmm6
+	paddq %xmm13, %xmm3
+	movdqa %xmm10, %xmm13
+	paddq %xmm14, %xmm4
+	movq -8(%rax), %xmm14
+	paddq %xmm8, %xmm2
+	movq -32(%rax), %xmm8
+	pmuludq %xmm12, %xmm13
+	paddq %xmm15, %xmm3
+	pmuludq %xmm12, %xmm1
+	movdqa 216(%rsp), %xmm15
+	pmuludq 216(%rsp), %xmm10
+	punpcklqdq %xmm6, %xmm8
+	movq -24(%rax), %xmm6
+	pmuludq %xmm9, %xmm15
+	paddq %xmm13, %xmm0
+	movdqa 296(%rsp), %xmm13
+	paddq 248(%rsp), %xmm1
+	punpcklqdq %xmm14, %xmm6
+	movdqa 296(%rsp), %xmm14
+	pmuludq %xmm9, %xmm13
+	pmuludq 120(%rsp), %xmm9
+	movdqa %xmm15, 72(%rsp)
+	paddq %xmm10, %xmm1
+	movdqa 216(%rsp), %xmm15
+	pmuludq %xmm7, %xmm14
+	movdqa %xmm6, %xmm10
+	paddq %xmm9, %xmm1
+	pmuludq %xmm7, %xmm15
+	paddq %xmm13, %xmm0
+	paddq 72(%rsp), %xmm3
+	movdqa 120(%rsp), %xmm13
+	psllq $12, %xmm10
+	paddq %xmm14, %xmm2
+	movdqa %xmm5, %xmm14
+	pand %xmm8, %xmm14
+	pmuludq %xmm7, %xmm13
+	paddq %xmm15, %xmm0
+	movdqa %xmm14, 248(%rsp)
+	movdqa %xmm8, %xmm14
+	psrlq $52, %xmm8
+	movdqu (%rax), %xmm9
+	por %xmm10, %xmm8
+	pmuludq 24(%rsp), %xmm7
+	movdqu 16(%rax), %xmm10
+	paddq %xmm13, %xmm3
+	pxor %xmm13, %xmm13
+	movdqa %xmm9, %xmm15
+	paddq %xmm7, %xmm1
+	movdqa %xmm6, %xmm7
+	movdqa %xmm10, -72(%rsp)
+	punpckldq %xmm10, %xmm15
+	movdqa %xmm15, %xmm10
+	punpckldq %xmm13, %xmm10
+	punpckhdq -72(%rsp), %xmm9
+	psrlq $40, %xmm6
+	movdqa %xmm10, 72(%rsp)
+	movdqa %xmm9, %xmm10
+	punpckhdq %xmm13, %xmm9
+	psllq $18, %xmm9
+	paddq 72(%rsp), %xmm4
+	addq $64, %rax
+	paddq %xmm9, %xmm3
+	movdqa 40(%rsp), %xmm9
+	cmpq $63, %rcx
+	punpckhdq %xmm13, %xmm15
+	psllq $6, %xmm15
+	punpckldq %xmm13, %xmm10
+	paddq %xmm15, %xmm2
+	psllq $12, %xmm10
+	por 168(%rsp), %xmm6
+	pmuludq %xmm6, %xmm9
+	movdqa 88(%rsp), %xmm15
+	paddq %xmm10, %xmm0
+	movdqa 88(%rsp), %xmm13
+	psrlq $14, %xmm7
+	pand %xmm5, %xmm8
+	movdqa 184(%rsp), %xmm10
+	pand %xmm5, %xmm7
+	pmuludq %xmm7, %xmm15
+	paddq %xmm9, %xmm4
+	pmuludq %xmm6, %xmm13
+	movdqa 184(%rsp), %xmm9
+	paddq 168(%rsp), %xmm1
+	pmuludq %xmm7, %xmm10
+	pmuludq %xmm6, %xmm9
+	paddq %xmm15, %xmm4
+	movdqa 184(%rsp), %xmm15
+	paddq %xmm13, %xmm2
+	psrlq $26, %xmm14
+	movdqa 264(%rsp), %xmm13
+	paddq %xmm10, %xmm2
+	pmuludq %xmm8, %xmm15
+	pand %xmm5, %xmm14
+	paddq %xmm9, %xmm0
+	pmuludq %xmm6, %xmm13
+	movdqa 264(%rsp), %xmm9
+	movdqa 264(%rsp), %xmm10
+	pmuludq %xmm11, %xmm6
+	pmuludq %xmm8, %xmm9
+	paddq %xmm15, %xmm4
+	movdqa 264(%rsp), %xmm15
+	pmuludq %xmm14, %xmm10
+	paddq %xmm13, %xmm3
+	movdqa %xmm7, %xmm13
+	pmuludq %xmm7, %xmm15
+	paddq %xmm6, %xmm1
+	movdqa 312(%rsp), %xmm6
+	paddq %xmm9, %xmm2
+	pmuludq %xmm11, %xmm13
+	movdqa 248(%rsp), %xmm9
+	paddq %xmm10, %xmm4
+	pmuludq %xmm8, %xmm6
+	pmuludq 312(%rsp), %xmm7
+	paddq %xmm15, %xmm0
+	movdqa %xmm9, %xmm10
+	movdqa %xmm14, %xmm15
+	pmuludq %xmm11, %xmm10
+	paddq %xmm13, %xmm3
+	movdqa %xmm8, %xmm13
+	pmuludq %xmm11, %xmm13
+	paddq %xmm6, %xmm3
+	paddq %xmm7, %xmm1
+	movdqa 232(%rsp), %xmm6
+	pmuludq %xmm11, %xmm15
+	pmuludq 232(%rsp), %xmm8
+	paddq %xmm10, %xmm4
+	paddq %xmm8, %xmm1
+	movdqa 312(%rsp), %xmm10
+	paddq %xmm13, %xmm0
+	pmuludq %xmm14, %xmm6
+	movdqa 312(%rsp), %xmm13
+	pmuludq %xmm9, %xmm10
+	paddq %xmm15, %xmm2
+	movdqa 232(%rsp), %xmm7
+	pmuludq %xmm14, %xmm13
+	pmuludq 152(%rsp), %xmm14
+	paddq %xmm14, %xmm1
+	pmuludq %xmm9, %xmm7
+	paddq %xmm6, %xmm3
+	paddq %xmm10, %xmm2
+	movdqa 152(%rsp), %xmm10
+	paddq %xmm13, %xmm0
+	pmuludq %xmm9, %xmm10
+	paddq %xmm7, %xmm0
+	movdqa %xmm4, %xmm7
+	psrlq $26, %xmm7
+	pmuludq 56(%rsp), %xmm9
+	pand %xmm5, %xmm4
+	paddq %xmm7, %xmm2
+	paddq %xmm9, %xmm1
+	paddq %xmm10, %xmm3
+	movdqa %xmm2, %xmm7
+	movdqa %xmm2, %xmm9
+	movdqa %xmm3, %xmm6
+	psrlq $26, %xmm7
+	pand %xmm5, %xmm3
+	psrlq $26, %xmm6
+	paddq %xmm7, %xmm0
+	pand %xmm5, %xmm9
+	paddq %xmm6, %xmm1
+	movdqa %xmm0, %xmm10
+	movdqa %xmm1, %xmm6
+	pand %xmm5, %xmm10
+	pand %xmm5, %xmm1
+	psrlq $26, %xmm6
+	pmuludq 136(%rsp), %xmm6
+	paddq %xmm6, %xmm4
+	movdqa %xmm0, %xmm6
+	psrlq $26, %xmm6
+	movdqa %xmm4, %xmm2
+	movdqa %xmm4, %xmm7
+	paddq %xmm6, %xmm3
+	psrlq $26, %xmm2
+	pand %xmm5, %xmm7
+	movdqa %xmm3, %xmm0
+	paddq %xmm2, %xmm9
+	pand %xmm5, %xmm3
+	psrlq $26, %xmm0
+	paddq %xmm0, %xmm1
+	ja .Lpoly1305_blocks_x86_13
+	leaq -64(%rdx), %rax
+	movdqa %xmm3, %xmm6
+	andl $63, %edx
+	andq $-64, %rax
+	leaq 64(%rsi,%rax), %rsi
+.Lpoly1305_blocks_x86_7:
+	cmpq $31, %rdx
+	jbe .Lpoly1305_blocks_x86_9
+	movdqa -24(%rsp), %xmm13
+	movdqa %xmm6, %xmm0
+	movdqa %xmm6, %xmm3
+	movdqa 40(%rsp), %xmm11
+	movdqa %xmm1, %xmm12
+	testq %rsi, %rsi
+	movdqa -40(%rsp), %xmm2
+	pmuludq %xmm13, %xmm0
+	movdqa %xmm1, %xmm8
+	pmuludq %xmm1, %xmm11
+	movdqa %xmm10, %xmm4
+	movdqa %xmm1, %xmm14
+	pmuludq %xmm2, %xmm3
+	movdqa %xmm6, %xmm15
+	pmuludq %xmm1, %xmm13
+	movdqa %xmm7, %xmm1
+	pmuludq %xmm2, %xmm12
+	paddq %xmm0, %xmm11
+	movdqa -56(%rsp), %xmm0
+	pmuludq %xmm10, %xmm2
+	paddq %xmm3, %xmm13
+	pmuludq %xmm0, %xmm4
+	movdqa %xmm9, %xmm3
+	pmuludq %xmm0, %xmm3
+	paddq %xmm2, %xmm11
+	pmuludq %xmm0, %xmm8
+	movdqa %xmm6, %xmm2
+	pmuludq %xmm0, %xmm2
+	movdqa -8(%rsp), %xmm0
+	paddq %xmm4, %xmm13
+	movdqa 312(%rsp), %xmm4
+	paddq %xmm3, %xmm11
+	pmuludq 312(%rsp), %xmm6
+	movdqa 312(%rsp), %xmm3
+	pmuludq %xmm0, %xmm1
+	paddq %xmm2, %xmm12
+	pmuludq %xmm0, %xmm15
+	movdqa %xmm9, %xmm2
+	pmuludq %xmm0, %xmm2
+	pmuludq %xmm7, %xmm3
+	paddq %xmm1, %xmm11
+	movdqa 232(%rsp), %xmm1
+	pmuludq %xmm0, %xmm14
+	paddq %xmm15, %xmm8
+	pmuludq %xmm10, %xmm0
+	paddq %xmm2, %xmm13
+	movdqa 312(%rsp), %xmm2
+	pmuludq %xmm10, %xmm4
+	paddq %xmm3, %xmm13
+	movdqa 152(%rsp), %xmm3
+	pmuludq %xmm9, %xmm2
+	paddq %xmm6, %xmm14
+	pmuludq 232(%rsp), %xmm10
+	paddq %xmm0, %xmm12
+	pmuludq %xmm9, %xmm1
+	paddq %xmm10, %xmm14
+	movdqa 232(%rsp), %xmm0
+	pmuludq %xmm7, %xmm3
+	paddq %xmm4, %xmm8
+	pmuludq 152(%rsp), %xmm9
+	paddq %xmm2, %xmm12
+	paddq %xmm9, %xmm14
+	pmuludq %xmm7, %xmm0
+	paddq %xmm1, %xmm8
+	pmuludq 56(%rsp), %xmm7
+	paddq %xmm3, %xmm8
+	paddq %xmm7, %xmm14
+	paddq %xmm0, %xmm12
+	je .Lpoly1305_blocks_x86_10
+	movdqu (%rsi), %xmm1
+	pxor %xmm0, %xmm0
+	paddq 168(%rsp), %xmm14
+	movdqu 16(%rsi), %xmm2
+	movdqa %xmm1, %xmm3
+	punpckldq %xmm2, %xmm3
+	punpckhdq %xmm2, %xmm1
+	movdqa %xmm3, %xmm4
+	movdqa %xmm1, %xmm2
+	punpckldq %xmm0, %xmm4
+	punpckhdq %xmm0, %xmm3
+	punpckhdq %xmm0, %xmm1
+	punpckldq %xmm0, %xmm2
+	movdqa %xmm2, %xmm0
+	psllq $6, %xmm3
+	paddq %xmm4, %xmm11
+	psllq $12, %xmm0
+	paddq %xmm3, %xmm13
+	psllq $18, %xmm1
+	paddq %xmm0, %xmm12
+	paddq %xmm1, %xmm8
+.Lpoly1305_blocks_x86_10:
+	movdqa %xmm11, %xmm9
+	movdqa %xmm8, %xmm1
+	movdqa %xmm11, %xmm7
+	psrlq $26, %xmm9
+	movdqa %xmm8, %xmm6
+	pand %xmm5, %xmm7
+	paddq %xmm13, %xmm9
+	psrlq $26, %xmm1
+	pand %xmm5, %xmm6
+	movdqa %xmm9, %xmm10
+	paddq %xmm14, %xmm1
+	pand %xmm5, %xmm9
+	psrlq $26, %xmm10
+	movdqa %xmm1, %xmm0
+	pand %xmm5, %xmm1
+	paddq %xmm12, %xmm10
+	psrlq $26, %xmm0
+	pmuludq 136(%rsp), %xmm0
+	movdqa %xmm10, %xmm2
+	paddq %xmm0, %xmm7
+	psrlq $26, %xmm2
+	movdqa %xmm7, %xmm0
+	pand %xmm5, %xmm10
+	paddq %xmm2, %xmm6
+	psrlq $26, %xmm0
+	pand %xmm5, %xmm7
+	movdqa %xmm6, %xmm2
+	paddq %xmm0, %xmm9
+	pand %xmm5, %xmm6
+	psrlq $26, %xmm2
+	paddq %xmm2, %xmm1
+.Lpoly1305_blocks_x86_9:
+	testq %rsi, %rsi
+	je .Lpoly1305_blocks_x86_11
+	movdqa %xmm7, 0(%rdi)
+	movdqa %xmm9, 16(%rdi)
+	movdqa %xmm10, 32(%rdi)
+	movdqa %xmm6, 48(%rdi)
+	movdqa %xmm1, 64(%rdi)
+	movq -8(%rbp), %rbx
+	leave
+	ret
+.Lpoly1305_blocks_x86_5:
+	movdqa 0(%rdi), %xmm7
+	movdqa 16(%rdi), %xmm9
+	movdqa 32(%rdi), %xmm10
+	movdqa 48(%rdi), %xmm6
+	movdqa 64(%rdi), %xmm1
+	jmp .Lpoly1305_blocks_x86_6
+.Lpoly1305_blocks_x86_11:
+	movdqa %xmm7, %xmm0
+	movdqa %xmm9, %xmm2
+	movdqa %xmm6, %xmm3
+	psrldq $8, %xmm0
+	movabsq $4398046511103, %rbx
+	paddq %xmm0, %xmm7
+	psrldq $8, %xmm2
+	movdqa %xmm10, %xmm0
+	movd %xmm7, %edx
+	paddq %xmm2, %xmm9
+	psrldq $8, %xmm0
+	movl %edx, %ecx
+	movd %xmm9, %eax
+	paddq %xmm0, %xmm10
+	shrl $26, %ecx
+	psrldq $8, %xmm3
+	movdqa %xmm1, %xmm0
+	addl %ecx, %eax
+	movd %xmm10, %ecx
+	paddq %xmm3, %xmm6
+	movl %eax, %r9d
+	shrl $26, %eax
+	psrldq $8, %xmm0
+	addl %ecx, %eax
+	movd %xmm6, %ecx
+	paddq %xmm0, %xmm1
+	movl %eax, %esi
+	andl $67108863, %r9d
+	movd %xmm1, %r10d
+	shrl $26, %esi
+	andl $67108863, %eax
+	andl $67108863, %edx
+	addl %ecx, %esi
+	salq $8, %rax
+	movl %r9d, %ecx
+	shrl $18, %r9d
+	movl %esi, %r8d
+	shrl $26, %esi
+	andl $67108863, %r8d
+	addl %r10d, %esi
+	orq %r9, %rax
+	salq $16, %rsi
+	movq %r8, %r9
+	shrl $10, %r8d
+	salq $26, %rcx
+	orq %r8, %rsi
+	salq $34, %r9
+	orq %rdx, %rcx
+	movq %rsi, %r8
+	shrq $42, %rsi
+	movabsq $17592186044415, %rdx
+	orq %r9, %rax
+	andq %rbx, %r8
+	leaq (%rsi,%rsi,4), %rsi
+	andq %rdx, %rcx
+	andq %rdx, %rax
+	movabsq $-4398046511104, %r10
+	addq %rsi, %rcx
+	movq %rcx, %rsi
+	shrq $44, %rcx
+	addq %rcx, %rax
+	andq %rdx, %rsi
+	movq %rax, %rcx
+	shrq $44, %rax
+	addq %r8, %rax
+	andq %rdx, %rcx
+	andq %rax, %rbx
+	shrq $42, %rax
+	leaq (%rsi,%rax,4), %rsi
+	addq %rbx, %r10
+	addq %rax, %rsi
+	movq %rsi, %r8
+	shrq $44, %rsi
+	andq %rdx, %r8
+	addq %rcx, %rsi
+	leaq 5(%r8), %r9
+	movq %r9, %r11
+	andq %rdx, %r9
+	shrq $44, %r11
+	addq %rsi, %r11
+	movq %r11, %rax
+	andq %r11, %rdx
+	shrq $44, %rax
+	addq %rax, %r10
+	movq %r10, %rax
+	shrq $63, %rax
+	subq $1, %rax
+	movq %rax, %rcx
+	andq %rax, %r9
+	andq %rax, %rdx
+	notq %rcx
+	andq %r10, %rax
+	andq %rcx, %r8
+	andq %rcx, %rsi
+	andq %rbx, %rcx
+	orq %r9, %r8
+	orq %rdx, %rsi
+	orq %rax, %rcx
+	movq %r8, 0(%rdi)
+	movq %rsi, 8(%rdi)
+	movq %rcx, 16(%rdi)
+	movq -8(%rbp), %rbx
+	movq %rbp, %rax
+	subq %rsp, %rax
+	pxor %xmm15, %xmm15
+	pxor %xmm7, %xmm7
+	pxor %xmm14, %xmm14
+	pxor %xmm6, %xmm6
+	pxor %xmm13, %xmm13
+	pxor %xmm5, %xmm5
+	pxor %xmm12, %xmm12
+	pxor %xmm4, %xmm4
+	leave
+	addq $8, %rax
+	pxor %xmm11, %xmm11
+	pxor %xmm3, %xmm3
+	pxor %xmm10, %xmm10
+	pxor %xmm2, %xmm2
+	pxor %xmm9, %xmm9
+	pxor %xmm1, %xmm1
+	pxor %xmm8, %xmm8
+	pxor %xmm0, %xmm0
+	ret
+.size _gcry_poly1305_amd64_sse2_blocks,.-_gcry_poly1305_amd64_sse2_blocks;
+
+#endif
diff --git a/cipher/poly1305.c b/cipher/poly1305.c
index 472ae42c..cd1902aa 100644
--- a/cipher/poly1305.c
+++ b/cipher/poly1305.c
@@ -38,6 +38,25 @@ static const char *selftest (void);
 
 
 
+#ifdef POLY1305_USE_SSE2
+
+void _gcry_poly1305_amd64_sse2_init_ext(void *state, const poly1305_key_t *key);
+unsigned int _gcry_poly1305_amd64_sse2_finish_ext(void *state, const byte *m,
+						  size_t remaining,
+						  byte mac[16]);
+unsigned int _gcry_poly1305_amd64_sse2_blocks(void *ctx, const byte *m,
+					      size_t bytes);
+
+static const poly1305_ops_t poly1305_amd64_sse2_ops = {
+  POLY1305_SSE2_BLOCKSIZE,
+  _gcry_poly1305_amd64_sse2_init_ext,
+  _gcry_poly1305_amd64_sse2_blocks,
+  _gcry_poly1305_amd64_sse2_finish_ext
+};
+
+#endif
+
+
 #ifdef HAVE_U64_TYPEDEF
 
 /* Reference unoptimized poly1305 implementation using 32 bit * 32 bit = 64 bit
@@ -612,7 +631,11 @@ _gcry_poly1305_init (poly1305_context_t * ctx, const byte * key,
   if (selftest_failed)
     return GPG_ERR_SELFTEST_FAILED;
 
+#ifdef POLY1305_USE_SSE2
+  ctx->ops = &poly1305_amd64_sse2_ops;
+#else
   ctx->ops = &poly1305_default_ops;
+#endif
 
   buf_cpy (keytmp.b, key, POLY1305_KEYLEN);
   poly1305_init (ctx, &keytmp);
diff --git a/configure.ac b/configure.ac
index 3a0fd524..4dc36d5f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1821,6 +1821,13 @@ if test "$found" = "1" ; then
    esac
 fi
 
+case "${host}" in
+   x86_64-*-*)
+      # Build with the assembly implementation
+      GCRYPT_CIPHERS="$GCRYPT_CIPHERS poly1305-sse2-amd64.lo"
+   ;;
+esac
+
 LIST_MEMBER(dsa, $enabled_pubkey_ciphers)
 if test "$found" = "1" ; then
    GCRYPT_PUBKEY_CIPHERS="$GCRYPT_PUBKEY_CIPHERS dsa.lo"