Add bulk processing for hash transform functions

* cipher/hash-common.c (_gcry_md_block_write): Preload 'hd->blocksize'
to stack, pass number of blocks to 'hd->bwrite'.
* cipher/hash-common.c (_gcry_md_block_write_t): Add 'nblks'.
* cipher/gostr3411-94.c: Rename 'transform' function to
'transform_blk', add new 'transform' function with 'nblks' as
additional input.
* cipher/md4.c: Ditto.
* cipher/md5.c: Ditto.
* cipher/md4.c: Ditto.
* cipher/rmd160.c: Ditto.
* cipher/sha1.c: Ditto.
* cipher/sha256.c: Ditto.
* cipher/sha512.c: Ditto.
* cipher/stribog.c: Ditto.
* cipher/tiger.c: Ditto.
* cipher/whirlpool.c: Ditto.
--

Pass number of blocks to algorithm for futher optimizations.

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>

12 files changed, 208 insertions, 58 deletions

diff --git a/cipher/gostr3411-94.c b/cipher/gostr3411-94.c
index 12672165..b3326aa5 100644
--- a/cipher/gostr3411-94.c
+++ b/cipher/gostr3411-94.c
@@ -41,7 +41,7 @@ typedef struct {
 } GOSTR3411_CONTEXT;
 
 static unsigned int
-transform (void *c, const unsigned char *data);
+transform (void *c, const unsigned char *data, size_t nblks);
 
 static void
 gost3411_init (void *context)
@@ -211,7 +211,7 @@ do_hash_step (GOST28147_context *hd, unsigned char *h, unsigned char *m)
 
 
 static unsigned int
-transform (void *ctx, const unsigned char *data)
+transform_blk (void *ctx, const unsigned char *data)
 {
   GOSTR3411_CONTEXT *hd = ctx;
   byte m[32];
@@ -224,6 +224,23 @@ transform (void *ctx, const unsigned char *data)
   return /* burn_stack */ burn + 3 * sizeof(void*) + 32 + 2 * sizeof(void*);
 }
 
+
+static unsigned int
+transform ( void *c, const unsigned char *data, size_t nblks )
+{
+  unsigned int burn;
+
+  do
+    {
+      burn = transform_blk (c, data);
+      data += 32;
+    }
+  while (--nblks);
+
+  return burn;
+}
+
+
 /*
    The routine finally terminates the computation and returns the
    digest.  The handle is prepared for a new cycle, but adding bytes
diff --git a/cipher/hash-common.c b/cipher/hash-common.c
index ffbc39ed..ed63a0ba 100644
--- a/cipher/hash-common.c
+++ b/cipher/hash-common.c
@@ -102,16 +102,18 @@ _gcry_md_block_write (void *context, const void *inbuf_arg, size_t inlen)
   const unsigned char *inbuf = inbuf_arg;
   gcry_md_block_ctx_t *hd = context;
   unsigned int stack_burn = 0;
+  const unsigned int blocksize = hd->blocksize;
+  size_t inblocks;
 
-  if (sizeof(hd->buf) < hd->blocksize)
+  if (sizeof(hd->buf) < blocksize)
     BUG();
 
   if (hd->buf == NULL || hd->bwrite == NULL)
     return;
 
-  if (hd->count == hd->blocksize)  /* Flush the buffer. */
+  if (hd->count == blocksize)  /* Flush the buffer. */
     {
-      stack_burn = hd->bwrite (hd, hd->buf);
+      stack_burn = hd->bwrite (hd, hd->buf, 1);
       _gcry_burn_stack (stack_burn);
       stack_burn = 0;
       hd->count = 0;
@@ -123,23 +125,24 @@ _gcry_md_block_write (void *context, const void *inbuf_arg, size_t inlen)
 
   if (hd->count)
     {
-      for (; inlen && hd->count < hd->blocksize; inlen--)
+      for (; inlen && hd->count < blocksize; inlen--)
         hd->buf[hd->count++] = *inbuf++;
       _gcry_md_block_write (hd, NULL, 0);
       if (!inlen)
         return;
     }
 
-  while (inlen >= hd->blocksize)
+  if (inlen >= blocksize)
     {
-      stack_burn = hd->bwrite (hd, inbuf);
+      inblocks = inlen / blocksize;
+      stack_burn = hd->bwrite (hd, inbuf, inblocks);
       hd->count = 0;
-      if (!++hd->nblocks)
-        hd->nblocks_high++;
-      inlen -= hd->blocksize;
-      inbuf += hd->blocksize;
+      hd->nblocks_high += (hd->nblocks + inblocks < inblocks);
+      hd->nblocks += inblocks;
+      inlen -= inblocks * blocksize;
+      inbuf += inblocks * blocksize;
     }
   _gcry_burn_stack (stack_burn);
-  for (; inlen && hd->count < hd->blocksize; inlen--)
+  for (; inlen && hd->count < blocksize; inlen--)
     hd->buf[hd->count++] = *inbuf++;
 }
diff --git a/cipher/hash-common.h b/cipher/hash-common.h
index aa95365a..c9491913 100644
--- a/cipher/hash-common.h
+++ b/cipher/hash-common.h
@@ -30,7 +30,8 @@ const char * _gcry_hash_selftest_check_one
 
 /* Type for the md_write helper function.  */
 typedef unsigned int (*_gcry_md_block_write_t) (void *c,
-						const unsigned char *buf);
+						const unsigned char *blks,
+						size_t nblks);
 
 #if defined(HAVE_U64_TYPEDEF) && (defined(USE_SHA512) || defined(USE_WHIRLPOOL))
 /* SHA-512 needs u64 and larger buffer. Whirlpool needs u64. */
diff --git a/cipher/md4.c b/cipher/md4.c
index b9a1a95a..40dc0586 100644
--- a/cipher/md4.c
+++ b/cipher/md4.c
@@ -66,7 +66,7 @@ typedef struct {
 } MD4_CONTEXT;
 
 static unsigned int
-transform ( void *c, const unsigned char *data );
+transform ( void *c, const unsigned char *data, size_t nblks );
 
 static void
 md4_init( void *context )
@@ -94,7 +94,7 @@ md4_init( void *context )
  * transform 64 bytes
  */
 static unsigned int
-transform ( void *c, const unsigned char *data )
+transform_blk ( void *c, const unsigned char *data )
 {
   MD4_CONTEXT *ctx = c;
   u32 in[16];
@@ -181,6 +181,21 @@ transform ( void *c, const unsigned char *data )
 }
 
 
+static unsigned int
+transform ( void *c, const unsigned char *data, size_t nblks )
+{
+  unsigned int burn;
+
+  do
+    {
+      burn = transform_blk (c, data);
+      data += 64;
+    }
+  while (--nblks);
+
+  return burn;
+}
+
 
 /* The routine final terminates the message-digest computation and
  * ends with the desired message digest in mdContext->digest[0...15].
@@ -234,7 +249,7 @@ md4_final( void *context )
   /* append the 64 bit count */
   buf_put_le32(hd->bctx.buf + 56, lsb);
   buf_put_le32(hd->bctx.buf + 60, msb);
-  burn = transform( hd, hd->bctx.buf );
+  burn = transform ( hd, hd->bctx.buf, 1 );
   _gcry_burn_stack (burn);
 
   p = hd->bctx.buf;
diff --git a/cipher/md5.c b/cipher/md5.c
index 79b6e871..d06d3f78 100644
--- a/cipher/md5.c
+++ b/cipher/md5.c
@@ -50,7 +50,7 @@ typedef struct {
 } MD5_CONTEXT;
 
 static unsigned int
-transform ( void *ctx, const unsigned char *data );
+transform ( void *ctx, const unsigned char *data, size_t datalen );
 
 static void
 md5_init( void *context )
@@ -81,10 +81,10 @@ md5_init( void *context )
 
 
 /****************
- * transform n*64 bytes
+ * transform 64 bytes
  */
 static unsigned int
-transform ( void *c, const unsigned char *data )
+transform_blk ( void *c, const unsigned char *data )
 {
   MD5_CONTEXT *ctx = c;
   u32 correct_words[16];
@@ -205,6 +205,21 @@ transform ( void *c, const unsigned char *data )
 }
 
 
+static unsigned int
+transform ( void *c, const unsigned char *data, size_t nblks )
+{
+  unsigned int burn;
+
+  do
+    {
+      burn = transform_blk (c, data);
+      data += 64;
+    }
+  while (--nblks);
+
+  return burn;
+}
+
 
 /* The routine final terminates the message-digest computation and
  * ends with the desired message digest in mdContext->digest[0...15].
@@ -258,7 +273,7 @@ md5_final( void *context)
   /* append the 64 bit count */
   buf_put_le32(hd->bctx.buf + 56, lsb);
   buf_put_le32(hd->bctx.buf + 60, msb);
-  burn = transform( hd, hd->bctx.buf );
+  burn = transform ( hd, hd->bctx.buf, 1 );
   _gcry_burn_stack (burn);
 
   p = hd->bctx.buf;
diff --git a/cipher/rmd160.c b/cipher/rmd160.c
index a6d9a80a..224694f2 100644
--- a/cipher/rmd160.c
+++ b/cipher/rmd160.c
@@ -141,7 +141,7 @@
  */
 
 static unsigned int
-transform ( void *ctx, const unsigned char *data );
+transform ( void *ctx, const unsigned char *data, size_t nblks );
 
 void
 _gcry_rmd160_init (void *context)
@@ -167,7 +167,7 @@ _gcry_rmd160_init (void *context)
  * Transform the message X which consists of 16 32-bit-words
  */
 static unsigned int
-transform ( void *ctx, const unsigned char *data )
+transform_blk ( void *ctx, const unsigned char *data )
 {
   RMD160_CONTEXT *hd = ctx;
   register u32 a,b,c,d,e;
@@ -386,6 +386,22 @@ transform ( void *ctx, const unsigned char *data )
 }
 
 
+static unsigned int
+transform ( void *c, const unsigned char *data, size_t nblks )
+{
+  unsigned int burn;
+
+  do
+    {
+      burn = transform_blk (c, data);
+      data += 64;
+    }
+  while (--nblks);
+
+  return burn;
+}
+
+
 /****************
  * Apply the rmd160 transform function on the buffer which must have
  * a length 64 bytes. Do not use this function together with the
@@ -397,7 +413,7 @@ _gcry_rmd160_mixblock ( RMD160_CONTEXT *hd, void *blockof64byte )
 {
   char *p = blockof64byte;
 
-  transform ( hd, blockof64byte );
+  transform ( hd, blockof64byte, 64 );
 #define X(a) do { *(u32*)p = hd->h##a ; p += 4; } while(0)
   X(0);
   X(1);
@@ -457,7 +473,7 @@ rmd160_final( void *context )
   /* append the 64 bit count */
   buf_put_le32(hd->bctx.buf + 56, lsb);
   buf_put_le32(hd->bctx.buf + 60, msb);
-  burn = transform( hd, hd->bctx.buf );
+  burn = transform ( hd, hd->bctx.buf, 1 );
   _gcry_burn_stack (burn);
 
   p = hd->bctx.buf;
diff --git a/cipher/sha1.c b/cipher/sha1.c
index 18b6daa1..53f75381 100644
--- a/cipher/sha1.c
+++ b/cipher/sha1.c
@@ -70,7 +70,7 @@ typedef struct
 } SHA1_CONTEXT;
 
 static unsigned int
-transform (void *c, const unsigned char *data);
+transform (void *c, const unsigned char *data, size_t nblks);
 
 
 static void
@@ -122,7 +122,7 @@ sha1_init (void *context)
  * Transform NBLOCKS of each 64 bytes (16 32-bit words) at DATA.
  */
 static unsigned int
-_transform (void *ctx, const unsigned char *data)
+transform_blk (void *ctx, const unsigned char *data)
 {
   SHA1_CONTEXT *hd = ctx;
   const u32 *idata = (const void *)data;
@@ -239,17 +239,33 @@ _gcry_sha1_transform_amd64_ssse3 (void *state, const unsigned char *data);
 
 
 static unsigned int
-transform (void *ctx, const unsigned char *data)
+transform (void *ctx, const unsigned char *data, size_t nblks)
 {
   SHA1_CONTEXT *hd = ctx;
+  unsigned int burn;
 
 #ifdef USE_SSSE3
   if (hd->use_ssse3)
-    return _gcry_sha1_transform_amd64_ssse3 (&hd->h0, data)
-           + 4 * sizeof(void*);
+    {
+      do
+        {
+          burn = _gcry_sha1_transform_amd64_ssse3 (&hd->h0, data);
+          data += 64;
+        }
+      while (--nblks);
+
+      return burn + 4 * sizeof(void*);
+    }
 #endif
 
-  return _transform (hd, data);
+  do
+    {
+      burn = transform_blk (ctx, data);
+      data += 64;
+    }
+  while (--nblks);
+
+  return burn;
 }
 
 
@@ -306,7 +322,7 @@ sha1_final(void *context)
   /* append the 64 bit count */
   buf_put_be32(hd->bctx.buf + 56, msb);
   buf_put_be32(hd->bctx.buf + 60, lsb);
-  burn = transform( hd, hd->bctx.buf );
+  burn = transform( hd, hd->bctx.buf, 1 );
   _gcry_burn_stack (burn);
 
   p = hd->bctx.buf;
diff --git a/cipher/sha256.c b/cipher/sha256.c
index f3c1d62c..c2045b8f 100644
--- a/cipher/sha256.c
+++ b/cipher/sha256.c
@@ -66,7 +66,7 @@ typedef struct {
 
 
 static unsigned int
-transform (void *c, const unsigned char *data);
+transform (void *c, const unsigned char *data, size_t nblks);
 
 
 static void
@@ -170,7 +170,7 @@ Sum1 (u32 x)
 
 
 static unsigned int
-_transform (void *ctx, const unsigned char *data)
+transform_blk (void *ctx, const unsigned char *data)
 {
   SHA256_CONTEXT *hd = ctx;
   static const u32 K[64] = {
@@ -283,17 +283,25 @@ unsigned int _gcry_sha256_transform_amd64_ssse3(const void *input_data,
 
 
 static unsigned int
-transform (void *ctx, const unsigned char *data)
+transform (void *ctx, const unsigned char *data, size_t nblks)
 {
   SHA256_CONTEXT *hd = ctx;
+  unsigned int burn;
 
 #ifdef USE_SSSE3
   if (hd->use_ssse3)
-    return _gcry_sha256_transform_amd64_ssse3 (data, &hd->h0, 1)
+    return _gcry_sha256_transform_amd64_ssse3 (data, &hd->h0, nblks)
            + 4 * sizeof(void*);
 #endif
 
-  return _transform (hd, data);
+  do
+    {
+      burn = transform_blk (hd, data);
+      data += 64;
+    }
+  while (--nblks);
+
+  return burn;
 }
 
 
@@ -348,7 +356,7 @@ sha256_final(void *context)
   /* append the 64 bit count */
   buf_put_be32(hd->bctx.buf + 56, msb);
   buf_put_be32(hd->bctx.buf + 60, lsb);
-  burn = transform (hd, hd->bctx.buf);
+  burn = transform (hd, hd->bctx.buf, 1);
   _gcry_burn_stack (burn);
 
   p = hd->bctx.buf;
diff --git a/cipher/sha512.c b/cipher/sha512.c
index 586c8097..215e8edf 100644
--- a/cipher/sha512.c
+++ b/cipher/sha512.c
@@ -116,7 +116,7 @@ typedef struct
 } SHA512_CONTEXT;
 
 static unsigned int
-transform (void *context, const unsigned char *data);
+transform (void *context, const unsigned char *data, size_t nblks);
 
 static void
 sha512_init (void *context)
@@ -273,7 +273,7 @@ static const u64 k[] =
  * Transform the message W which consists of 16 64-bit-words
  */
 static unsigned int
-__transform (SHA512_STATE *hd, const unsigned char *data)
+transform_blk (SHA512_STATE *hd, const unsigned char *data)
 {
   u64 a, b, c, d, e, f, g, h;
   u64 w[16];
@@ -561,32 +561,38 @@ unsigned int _gcry_sha512_transform_amd64_avx2(const void *input_data,
 
 
 static unsigned int
-transform (void *context, const unsigned char *data)
+transform (void *context, const unsigned char *data, size_t nblks)
 {
   SHA512_CONTEXT *ctx = context;
+  unsigned int burn;
 
 #ifdef USE_AVX2
   if (ctx->use_avx2)
-    return _gcry_sha512_transform_amd64_avx2 (data, &ctx->state, 1)
+    return _gcry_sha512_transform_amd64_avx2 (data, &ctx->state, nblks)
            + 4 * sizeof(void*);
 #endif
 
 #ifdef USE_AVX
   if (ctx->use_avx)
-    return _gcry_sha512_transform_amd64_avx (data, &ctx->state, 1)
+    return _gcry_sha512_transform_amd64_avx (data, &ctx->state, nblks)
            + 4 * sizeof(void*);
 #endif
 
 #ifdef USE_SSSE3
   if (ctx->use_ssse3)
-    return _gcry_sha512_transform_amd64_ssse3 (data, &ctx->state, 1)
+    return _gcry_sha512_transform_amd64_ssse3 (data, &ctx->state, nblks)
            + 4 * sizeof(void*);
 #endif
 
 #ifdef USE_ARM_NEON_ASM
   if (ctx->use_neon)
     {
-      _gcry_sha512_transform_armv7_neon (&ctx->state, data, k);
+      do
+        {
+          _gcry_sha512_transform_armv7_neon (&ctx->state, data, k);
+          data += 128;
+        }
+      while (--nblks);
 
       /* _gcry_sha512_transform_armv7_neon does not store sensitive data
        * to stack.  */
@@ -594,7 +600,14 @@ transform (void *context, const unsigned char *data)
     }
 #endif
 
-  return __transform (&ctx->state, data) + 3 * sizeof(void*);
+  do
+    {
+      burn = transform_blk (&ctx->state, data) + 3 * sizeof(void*);
+      data += 128;
+    }
+  while (--nblks);
+
+  return burn;
 }
 
 
@@ -652,7 +665,7 @@ sha512_final (void *context)
   /* append the 128 bit count */
   buf_put_be64(hd->bctx.buf + 112, msb);
   buf_put_be64(hd->bctx.buf + 120, lsb);
-  stack_burn_depth = transform (hd, hd->bctx.buf);
+  stack_burn_depth = transform (hd, hd->bctx.buf, 1);
   _gcry_burn_stack (stack_burn_depth);
 
   p = hd->bctx.buf;
diff --git a/cipher/stribog.c b/cipher/stribog.c
index a3cab362..6d1d3422 100644
--- a/cipher/stribog.c
+++ b/cipher/stribog.c
@@ -1194,7 +1194,7 @@ static inline void g (u64 *h, u64 *m, u64 *N)
 
 
 static unsigned int
-transform64 (void *context, const unsigned char *inbuf_arg);
+transform (void *context, const unsigned char *inbuf_arg, size_t datalen);
 
 
 static void
@@ -1205,7 +1205,7 @@ stribog_init_512 (void *context)
   memset (hd, 0, sizeof (*hd));
 
   hd->bctx.blocksize = 64;
-  hd->bctx.bwrite = transform64;
+  hd->bctx.bwrite = transform;
 }
 
 static void
@@ -1217,7 +1217,7 @@ stribog_init_256 (void *context)
 }
 
 static void
-transform (STRIBOG_CONTEXT *hd, const unsigned char *data, unsigned count)
+transform_bits (STRIBOG_CONTEXT *hd, const unsigned char *data, unsigned count)
 {
   u64 M[8];
   u64 l;
@@ -1248,15 +1248,30 @@ transform (STRIBOG_CONTEXT *hd, const unsigned char *data, unsigned count)
 }
 
 static unsigned int
-transform64 (void *context, const unsigned char *inbuf_arg)
+transform_blk (void *context, const unsigned char *inbuf_arg)
 {
   STRIBOG_CONTEXT *hd = context;
 
-  transform (hd, inbuf_arg, 64 * 8);
+  transform_bits (hd, inbuf_arg, 64 * 8);
 
   return /* burn_stack */ 768;
 }
 
+static unsigned int
+transform ( void *c, const unsigned char *data, size_t nblks )
+{
+  unsigned int burn;
+
+  do
+    {
+      burn = transform_blk (c, data);
+      data += 64;
+    }
+  while (--nblks);
+
+  return burn;
+}
+
 /*
    The routine finally terminates the computation and returns the
    digest.  The handle is prepared for a new cycle, but adding bytes
@@ -1276,7 +1291,7 @@ stribog_final (void *context)
   hd->bctx.buf[i++] = 1;
   while (i < 64)
     hd->bctx.buf[i++] = 0;
-  transform (hd, hd->bctx.buf, hd->bctx.count * 8);
+  transform_bits (hd, hd->bctx.buf, hd->bctx.count * 8);
 
   g (hd->h, hd->N, Z);
   g (hd->h, hd->Sigma, Z);
diff --git a/cipher/tiger.c b/cipher/tiger.c
index 9b8d0ef3..17c41197 100644
--- a/cipher/tiger.c
+++ b/cipher/tiger.c
@@ -590,7 +590,7 @@ static u64 sbox4[256] = {
 };
 
 static unsigned int
-transform ( void *ctx, const unsigned char *data );
+transform ( void *ctx, const unsigned char *data, size_t nblks );
 
 static void
 do_init (void *context, int variant)
@@ -695,7 +695,7 @@ key_schedule( u64 *x )
  * Transform the message DATA which consists of 512 bytes (8 words)
  */
 static unsigned int
-transform ( void *ctx, const unsigned char *data )
+transform_blk ( void *ctx, const unsigned char *data )
 {
   TIGER_CONTEXT *hd = ctx;
   u64 a,b,c,aa,bb,cc;
@@ -729,6 +729,22 @@ transform ( void *ctx, const unsigned char *data )
 }
 
 
+static unsigned int
+transform ( void *c, const unsigned char *data, size_t nblks )
+{
+  unsigned int burn;
+
+  do
+    {
+      burn = transform_blk (c, data);
+      data += 64;
+    }
+  while (--nblks);
+
+  return burn;
+}
+
+
 
 /* The routine terminates the computation
  */
@@ -779,7 +795,7 @@ tiger_final( void *context )
   /* append the 64 bit count */
   buf_put_le32(hd->bctx.buf + 56, lsb);
   buf_put_le32(hd->bctx.buf + 60, msb);
-  burn = transform( hd, hd->bctx.buf );
+  burn = transform( hd, hd->bctx.buf, 1 );
   _gcry_burn_stack (burn);
 
   p = hd->bctx.buf;
diff --git a/cipher/whirlpool.c b/cipher/whirlpool.c
index e562781d..57ca8826 100644
--- a/cipher/whirlpool.c
+++ b/cipher/whirlpool.c
@@ -1161,7 +1161,7 @@ static const u64 C7[256] =
 
 
 static unsigned int
-whirlpool_transform (void *ctx, const unsigned char *data);
+whirlpool_transform (void *ctx, const unsigned char *data, size_t nblks);
 
 
 
@@ -1181,7 +1181,7 @@ whirlpool_init (void *ctx)
  * Transform block.
  */
 static unsigned int
-whirlpool_transform (void *ctx, const unsigned char *data)
+whirlpool_transform_blk (void *ctx, const unsigned char *data)
 {
   whirlpool_context_t *context = ctx;
   whirlpool_block_t data_block;
@@ -1280,6 +1280,21 @@ whirlpool_transform (void *ctx, const unsigned char *data)
                         4 * sizeof(void*);
 }
 
+static unsigned int
+whirlpool_transform ( void *c, const unsigned char *data, size_t nblks )
+{
+  unsigned int burn;
+
+  do
+    {
+      burn = whirlpool_transform_blk (c, data);
+      data += BLOCK_SIZE;
+    }
+  while (--nblks);
+
+  return burn;
+}
+
 static void
 whirlpool_write (void *ctx, const void *buffer, size_t buffer_n)
 {