Add Intel PCLMUL acceleration for GCM

* cipher/cipher-gcm.c (fillM): Rename...
(do_fillM): ...to this.
(ghash): Remove.
(fillM): New macro.
(GHASH): Use 'do_ghash' instead of 'ghash'.
[GCM_USE_INTEL_PCLMUL] (do_ghash_pclmul): New.
(ghash): New.
(setupM): New.
(_gcry_cipher_gcm_encrypt, _gcry_cipher_gcm_decrypt)
(_gcry_cipher_gcm_authenticate, _gcry_cipher_gcm_setiv)
(_gcry_cipher_gcm_tag): Use 'ghash' instead of 'GHASH' and
'c->u_mode.gcm.u_tag.tag' instead of 'c->u_tag.tag'.
* cipher/cipher-internal.h (GCM_USE_INTEL_PCLMUL): New.
(gcry_cipher_handle): Move 'u_tag' and 'gcm_table' under
'u_mode.gcm'.
* configure.ac (pclmulsupport, gcry_cv_gcc_inline_asm_pclmul): New.
* src/g10lib.h (HWF_INTEL_PCLMUL): New.
* src/global.c: Add "intel-pclmul".
* src/hwf-x86.c (detect_x86_gnuc): Add check for Intel PCLMUL.
--

Speed-up GCM for Intel CPUs.

Intel Haswell (x86-64):
Old:
AES     GCM enc |      5.17 ns/B     184.4 MiB/s     16.55 c/B
        GCM dec |      4.38 ns/B     218.0 MiB/s     14.00 c/B
       GCM auth |      3.17 ns/B     300.4 MiB/s     10.16 c/B
New:
AES     GCM enc |      3.01 ns/B     317.2 MiB/s      9.62 c/B
        GCM dec |      1.96 ns/B     486.9 MiB/s      6.27 c/B
       GCM auth |     0.848 ns/B    1124.8 MiB/s      2.71 c/B

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>

1 files changed, 35 insertions, 0 deletions

diff --git a/configure.ac b/configure.ac
index c4f8776b..6d403436 100644
--- a/configure.ac
+++ b/configure.ac
@@ -567,6 +567,14 @@ AC_ARG_ENABLE(aesni-support,
 	      aesnisupport=$enableval,aesnisupport=yes)
 AC_MSG_RESULT($aesnisupport)
 
+# Implementation of the --disable-pclmul-support switch.
+AC_MSG_CHECKING([whether PCLMUL support is requested])
+AC_ARG_ENABLE(pclmul-support,
+              AC_HELP_STRING([--disable-pclmul-support],
+                 [Disable support for the Intel PCLMUL instructions]),
+	      pclmulsupport=$enableval,pclmulsupport=yes)
+AC_MSG_RESULT($pclmulsupport)
+
 # Implementation of the --disable-drng-support switch.
 AC_MSG_CHECKING([whether DRNG support is requested])
 AC_ARG_ENABLE(drng-support,
@@ -991,6 +999,23 @@ fi
 
 
 #
+# Check whether GCC inline assembler supports PCLMUL instructions.
+#
+AC_CACHE_CHECK([whether GCC inline assembler supports PCLMUL instructions],
+       [gcry_cv_gcc_inline_asm_pclmul],
+       [gcry_cv_gcc_inline_asm_pclmul=no
+        AC_COMPILE_IFELSE([AC_LANG_SOURCE(
+          [[void a(void) {
+              __asm__("pclmulqdq \$0, %%xmm1, %%xmm3\n\t":::"cc");
+            }]])],
+          [gcry_cv_gcc_inline_asm_pclmul=yes])])
+if test "$gcry_cv_gcc_inline_asm_pclmul" = "yes" ; then
+   AC_DEFINE(HAVE_GCC_INLINE_ASM_PCLMUL,1,
+     [Defined if inline assembler supports PCLMUL instructions])
+fi
+
+
+#
 # Check whether GCC inline assembler supports AVX instructions
 #
 AC_CACHE_CHECK([whether GCC inline assembler supports AVX instructions],
@@ -1369,6 +1394,11 @@ if test x"$aesnisupport" = xyes ; then
     aesnisupport="no (unsupported by compiler)"
   fi
 fi
+if test x"$pclmulsupport" = xyes ; then
+  if test "$gcry_cv_gcc_inline_asm_pclmul" != "yes" ; then
+    pclmulsupport="no (unsupported by compiler)"
+  fi
+fi
 if test x"$avxsupport" = xyes ; then
   if test "$gcry_cv_gcc_inline_asm_avx" != "yes" ; then
     avxsupport="no (unsupported by compiler)"
@@ -1389,6 +1419,10 @@ if test x"$aesnisupport" = xyes ; then
   AC_DEFINE(ENABLE_AESNI_SUPPORT, 1,
             [Enable support for Intel AES-NI instructions.])
 fi
+if test x"$pclmulsupport" = xyes ; then
+  AC_DEFINE(ENABLE_PCLMUL_SUPPORT, 1,
+            [Enable support for Intel PCLMUL instructions.])
+fi
 if test x"$avxsupport" = xyes ; then
   AC_DEFINE(ENABLE_AVX_SUPPORT,1,
             [Enable support for Intel AVX instructions.])
@@ -1826,6 +1860,7 @@ GCRY_MSG_SHOW([Random number generator:  ],[$random])
 GCRY_MSG_SHOW([Using linux capabilities: ],[$use_capabilities])
 GCRY_MSG_SHOW([Try using Padlock crypto: ],[$padlocksupport])
 GCRY_MSG_SHOW([Try using AES-NI crypto:  ],[$aesnisupport])
+GCRY_MSG_SHOW([Try using Intel PCLMUL:   ],[$pclmulsupport])
 GCRY_MSG_SHOW([Try using DRNG (RDRAND):  ],[$drngsupport])
 GCRY_MSG_SHOW([Try using Intel AVX:      ],[$avxsupport])
 GCRY_MSG_SHOW([Try using Intel AVX2:     ],[$avx2support])