Add OCB bulk crypt/auth functions for AES/AES-NI

* cipher/cipher-internal.h (gcry_cipher_handle): Add bulk.ocb_crypt
and bulk.ocb_auth.
(_gcry_cipher_ocb_get_l): New prototype.
* cipher/cipher-ocb.c (get_l): Rename to ...
(_gcry_cipher_ocb_get_l): ... this.
(_gcry_cipher_ocb_authenticate, ocb_crypt): Use bulk function when
available.
* cipher/cipher.c (_gcry_cipher_open_internal): Setup OCB bulk
functions for AES.
* cipher/rijndael-aesni.c (get_l, aesni_ocb_enc, aes_ocb_dec)
(_gcry_aes_aesni_ocb_crypt, _gcry_aes_aesni_ocb_auth): New.
* cipher/rijndael.c [USE_AESNI] (_gcry_aes_aesni_ocb_crypt)
(_gcry_aes_aesni_ocb_auth): New prototypes.
(_gcry_aes_ocb_crypt, _gcry_aes_ocb_auth): New.
* src/cipher.h (_gcry_aes_ocb_crypt, _gcry_aes_ocb_auth): New
prototypes.
* tests/basic.c (check_ocb_cipher_largebuf): New.
(check_ocb_cipher): Add large buffer encryption/decryption test.
--

Patch adds bulk encryption/decryption/authentication code for AES-NI
accelerated AES.

Benchmark on Intel i5-4570 (3200 Mhz, turbo off):

Before:
 AES            |  nanosecs/byte   mebibytes/sec   cycles/byte
        OCB enc |      2.12 ns/B     449.7 MiB/s      6.79 c/B
        OCB dec |      2.12 ns/B     449.6 MiB/s      6.79 c/B
       OCB auth |      2.07 ns/B     459.9 MiB/s      6.64 c/B

After:
 AES            |  nanosecs/byte   mebibytes/sec   cycles/byte
        OCB enc |     0.292 ns/B    3262.5 MiB/s     0.935 c/B
        OCB dec |     0.297 ns/B    3212.2 MiB/s     0.950 c/B
       OCB auth |     0.260 ns/B    3666.1 MiB/s     0.832 c/B

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>

1 files changed, 55 insertions, 29 deletions

diff --git a/cipher/cipher-ocb.c b/cipher/cipher-ocb.c
index 62e79bbd..bc6fd87f 100644
--- a/cipher/cipher-ocb.c
+++ b/cipher/cipher-ocb.c
@@ -115,8 +115,8 @@ bit_copy (unsigned char *d, const unsigned char *s,
    every 65536-th block.  L_TMP is a helper buffer of size
    OCB_BLOCK_LEN which is used to hold the computation if not taken
    from the table.  */
-static const unsigned char *
-get_l (gcry_cipher_hd_t c, unsigned char *l_tmp, u64 n)
+const unsigned char *
+_gcry_cipher_ocb_get_l (gcry_cipher_hd_t c, unsigned char *l_tmp, u64 n)
 {
   int ntz = _gcry_ctz64 (n);
 
@@ -257,6 +257,15 @@ _gcry_cipher_ocb_authenticate (gcry_cipher_hd_t c, const unsigned char *abuf,
   if (!abuflen)
     return 0;
 
+  /* Use a bulk method if available.  */
+  if (abuflen >= OCB_BLOCK_LEN && c->bulk.ocb_auth)
+    {
+      size_t nblks = abuflen / OCB_BLOCK_LEN;
+      c->bulk.ocb_auth (c, abuf, nblks);
+      abuf += nblks * OCB_BLOCK_LEN;
+      abuflen -= nblks * OCB_BLOCK_LEN;
+    }
+
   /* Hash all full blocks.  */
   while (abuflen >= OCB_BLOCK_LEN)
     {
@@ -264,7 +273,8 @@ _gcry_cipher_ocb_authenticate (gcry_cipher_hd_t c, const unsigned char *abuf,
 
       /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
       buf_xor_1 (c->u_mode.ocb.aad_offset,
-                 get_l (c, l_tmp, c->u_mode.ocb.aad_nblocks), OCB_BLOCK_LEN);
+                 _gcry_cipher_ocb_get_l (c, l_tmp, c->u_mode.ocb.aad_nblocks),
+                 OCB_BLOCK_LEN);
       /* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i)  */
       buf_xor (l_tmp, c->u_mode.ocb.aad_offset, abuf, OCB_BLOCK_LEN);
       c->spec->encrypt (&c->context.c, l_tmp, l_tmp);
@@ -341,40 +351,56 @@ ocb_crypt (gcry_cipher_hd_t c, int encrypt,
   else if ((inbuflen % OCB_BLOCK_LEN))
     return GPG_ERR_INV_LENGTH;  /* We support only full blocks for now.  */
 
-  if (encrypt)
+  /* Use a bulk method if available.  */
+  if (nblks && c->bulk.ocb_crypt)
     {
-      /* Checksum_i = Checksum_{i-1} xor P_i  */
-      ocb_checksum (c->u_ctr.ctr, inbuf, nblks);
+      c->bulk.ocb_crypt (c, outbuf, inbuf, nblks, encrypt);
+      inbuf  += nblks * OCB_BLOCK_LEN;
+      outbuf += nblks * OCB_BLOCK_LEN;
+      inbuflen -= nblks * OCB_BLOCK_LEN;
+      outbuflen -= nblks * OCB_BLOCK_LEN;
+      nblks = 0;
     }
 
-  /* Encrypt all full blocks.  */
-  while (inbuflen >= OCB_BLOCK_LEN)
+  if (nblks)
     {
-      c->u_mode.ocb.data_nblocks++;
+      gcry_cipher_encrypt_t crypt_fn =
+          encrypt ? c->spec->encrypt : c->spec->decrypt;
 
-      /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
-      buf_xor_1 (c->u_iv.iv,
-                 get_l (c, l_tmp, c->u_mode.ocb.data_nblocks), OCB_BLOCK_LEN);
-      /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i)  */
-      buf_xor (outbuf, c->u_iv.iv, inbuf, OCB_BLOCK_LEN);
       if (encrypt)
-        nburn = c->spec->encrypt (&c->context.c, outbuf, outbuf);
-      else
-        nburn = c->spec->decrypt (&c->context.c, outbuf, outbuf);
-      burn = nburn > burn ? nburn : burn;
-      buf_xor_1 (outbuf, c->u_iv.iv, OCB_BLOCK_LEN);
+        {
+          /* Checksum_i = Checksum_{i-1} xor P_i  */
+          ocb_checksum (c->u_ctr.ctr, inbuf, nblks);
+        }
 
-      inbuf += OCB_BLOCK_LEN;
-      inbuflen -= OCB_BLOCK_LEN;
-      outbuf += OCB_BLOCK_LEN;
-      outbuflen =- OCB_BLOCK_LEN;
-    }
+      /* Encrypt all full blocks.  */
+      while (inbuflen >= OCB_BLOCK_LEN)
+        {
+          c->u_mode.ocb.data_nblocks++;
+
+          /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
+          buf_xor_1 (c->u_iv.iv,
+                     _gcry_cipher_ocb_get_l (c, l_tmp,
+                                             c->u_mode.ocb.data_nblocks),
+                     OCB_BLOCK_LEN);
+          /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i)  */
+          buf_xor (outbuf, c->u_iv.iv, inbuf, OCB_BLOCK_LEN);
+          nburn = crypt_fn (&c->context.c, outbuf, outbuf);
+          burn = nburn > burn ? nburn : burn;
+          buf_xor_1 (outbuf, c->u_iv.iv, OCB_BLOCK_LEN);
+
+          inbuf += OCB_BLOCK_LEN;
+          inbuflen -= OCB_BLOCK_LEN;
+          outbuf += OCB_BLOCK_LEN;
+          outbuflen =- OCB_BLOCK_LEN;
+        }
 
-  if (!encrypt)
-    {
-      /* Checksum_i = Checksum_{i-1} xor P_i  */
-      ocb_checksum (c->u_ctr.ctr, outbuf - nblks * OCB_BLOCK_LEN, nblks);
-     }
+      if (!encrypt)
+        {
+          /* Checksum_i = Checksum_{i-1} xor P_i  */
+          ocb_checksum (c->u_ctr.ctr, outbuf - nblks * OCB_BLOCK_LEN, nblks);
+        }
+    }
 
   /* Encrypt final partial block.  Note that we expect INBUFLEN to be
      shorter than OCB_BLOCK_LEN (see above).  */