From 4e09aaa36d151c3312019724a77fc09aa345b82f Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Wed, 29 Apr 2015 18:18:07 +0300 Subject: Enable AES/AES-NI, AES/SSSE3 and GCM/PCLMUL implementations on WIN64 * cipher/cipher-gcm-intel-pclmul.c (_gcry_ghash_intel_pclmul) ( _gcry_ghash_intel_pclmul) [__WIN64__]: Store non-volatile vector registers before use and restore after. * cipher/cipher-internal.h (GCM_USE_INTEL_PCLMUL): Remove dependency on !defined(__WIN64__). * cipher/rijndael-aesni.c [__WIN64__] (aesni_prepare_2_6_variable, aesni_prepare, aesni_prepare_2_6, aesni_cleanup) ( aesni_cleanup_2_6): New. [!__WIN64__] (aesni_prepare_2_6_variable, aesni_prepare_2_6): New. (_gcry_aes_aesni_do_setkey, _gcry_aes_aesni_cbc_enc) (_gcry_aesni_ctr_enc, _gcry_aesni_cfb_dec, _gcry_aesni_cbc_dec) (_gcry_aesni_ocb_crypt, _gcry_aesni_ocb_auth): Use 'aesni_prepare_2_6'. * cipher/rijndael-internal.h (USE_SSSE3): Enable if HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS or HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS. (USE_AESNI): Remove dependency on !defined(__WIN64__) * cipher/rijndael-ssse3-amd64.c [HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS] (vpaes_ssse3_prepare, vpaes_ssse3_cleanup): New. [!HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS] (vpaes_ssse3_prepare): New. (vpaes_ssse3_prepare_enc, vpaes_ssse3_prepare_dec): Use 'vpaes_ssse3_prepare'. (_gcry_aes_ssse3_do_setkey, _gcry_aes_ssse3_prepare_decryption): Use 'vpaes_ssse3_prepare' and 'vpaes_ssse3_cleanup'. [HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS] (X): Add masking macro to exclude '.type' and '.size' markers from assembly code, as they are not support on WIN64/COFF objects. * configure.ac (gcry_cv_gcc_attribute_ms_abi) (gcry_cv_gcc_attribute_sysv_abi, gcry_cv_gcc_default_abi_is_ms_abi) (gcry_cv_gcc_default_abi_is_sysv_abi) (gcry_cv_gcc_win64_platform_as_ok): New checks. -- Signed-off-by: Jussi Kivilinna --- cipher/rijndael-aesni.c | 73 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 13 deletions(-) (limited to 'cipher/rijndael-aesni.c') diff --git a/cipher/rijndael-aesni.c b/cipher/rijndael-aesni.c index 147679f7..910bc681 100644 --- a/cipher/rijndael-aesni.c +++ b/cipher/rijndael-aesni.c @@ -49,24 +49,54 @@ typedef struct u128_s { u32 a, b, c, d; } u128_t; the use of these macros. There purpose is to make sure that the SSE regsiters are cleared and won't reveal any information about the key or the data. */ -#define aesni_prepare() do { } while (0) -#define aesni_cleanup() \ - do { asm volatile ("pxor %%xmm0, %%xmm0\n\t" \ - "pxor %%xmm1, %%xmm1\n" :: ); \ - } while (0) -#define aesni_cleanup_2_6() \ - do { asm volatile ("pxor %%xmm2, %%xmm2\n\t" \ - "pxor %%xmm3, %%xmm3\n" \ - "pxor %%xmm4, %%xmm4\n" \ - "pxor %%xmm5, %%xmm5\n" \ - "pxor %%xmm6, %%xmm6\n":: ); \ - } while (0) - +#ifdef __WIN64__ +/* XMM6-XMM15 are callee-saved registers on WIN64. */ +# define aesni_prepare_2_6_variable char win64tmp[16] +# define aesni_prepare() do { } while (0) +# define aesni_prepare_2_6() \ + do { asm volatile ("movdqu %%xmm6, %0\n\t" \ + : "=m" (*win64tmp) \ + : \ + : "memory"); \ + } while (0) +# define aesni_cleanup() \ + do { asm volatile ("pxor %%xmm0, %%xmm0\n\t" \ + "pxor %%xmm1, %%xmm1\n" :: ); \ + } while (0) +# define aesni_cleanup_2_6() \ + do { asm volatile ("movdqu %0, %%xmm6\n\t" \ + "pxor %%xmm2, %%xmm2\n" \ + "pxor %%xmm3, %%xmm3\n" \ + "pxor %%xmm4, %%xmm4\n" \ + "pxor %%xmm5, %%xmm5\n" \ + : \ + : "m" (*win64tmp) \ + : "memory"); \ + } while (0) +#else +# define aesni_prepare_2_6_variable +# define aesni_prepare() do { } while (0) +# define aesni_prepare_2_6() do { } while (0) +# define aesni_cleanup() \ + do { asm volatile ("pxor %%xmm0, %%xmm0\n\t" \ + "pxor %%xmm1, %%xmm1\n" :: ); \ + } while (0) +# define aesni_cleanup_2_6() \ + do { asm volatile ("pxor %%xmm2, %%xmm2\n\t" \ + "pxor %%xmm3, %%xmm3\n" \ + "pxor %%xmm4, %%xmm4\n" \ + "pxor %%xmm5, %%xmm5\n" \ + "pxor %%xmm6, %%xmm6\n":: ); \ + } while (0) +#endif void _gcry_aes_aesni_do_setkey (RIJNDAEL_context *ctx, const byte *key) { + aesni_prepare_2_6_variable; + aesni_prepare(); + aesni_prepare_2_6(); if (ctx->rounds < 12) { @@ -999,7 +1029,10 @@ _gcry_aes_aesni_cbc_enc (RIJNDAEL_context *ctx, unsigned char *outbuf, const unsigned char *inbuf, unsigned char *iv, size_t nblocks, int cbc_mac) { + aesni_prepare_2_6_variable; + aesni_prepare (); + aesni_prepare_2_6(); asm volatile ("movdqu %[iv], %%xmm5\n\t" : /* No output */ @@ -1044,8 +1077,10 @@ _gcry_aes_aesni_ctr_enc (RIJNDAEL_context *ctx, unsigned char *outbuf, { static const unsigned char be_mask[16] __attribute__ ((aligned (16))) = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 }; + aesni_prepare_2_6_variable; aesni_prepare (); + aesni_prepare_2_6(); asm volatile ("movdqa %[mask], %%xmm6\n\t" /* Preload mask */ "movdqa %[ctr], %%xmm5\n\t" /* Preload CTR */ @@ -1095,7 +1130,10 @@ _gcry_aes_aesni_cfb_dec (RIJNDAEL_context *ctx, unsigned char *outbuf, const unsigned char *inbuf, unsigned char *iv, size_t nblocks) { + aesni_prepare_2_6_variable; + aesni_prepare (); + aesni_prepare_2_6(); asm volatile ("movdqu %[iv], %%xmm6\n\t" : /* No output */ @@ -1177,7 +1215,10 @@ _gcry_aes_aesni_cbc_dec (RIJNDAEL_context *ctx, unsigned char *outbuf, const unsigned char *inbuf, unsigned char *iv, size_t nblocks) { + aesni_prepare_2_6_variable; + aesni_prepare (); + aesni_prepare_2_6(); asm volatile ("movdqu %[iv], %%xmm5\n\t" /* use xmm5 as fast IV storage */ @@ -1331,8 +1372,10 @@ aesni_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg, unsigned char *outbuf = outbuf_arg; const unsigned char *inbuf = inbuf_arg; u64 n = c->u_mode.ocb.data_nblocks; + aesni_prepare_2_6_variable; aesni_prepare (); + aesni_prepare_2_6 (); /* Preload Offset and Checksum */ asm volatile ("movdqu %[iv], %%xmm5\n\t" @@ -1473,8 +1516,10 @@ aesni_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg, unsigned char *outbuf = outbuf_arg; const unsigned char *inbuf = inbuf_arg; u64 n = c->u_mode.ocb.data_nblocks; + aesni_prepare_2_6_variable; aesni_prepare (); + aesni_prepare_2_6 (); /* Preload Offset and Checksum */ asm volatile ("movdqu %[iv], %%xmm5\n\t" @@ -1625,8 +1670,10 @@ _gcry_aes_aesni_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, RIJNDAEL_context *ctx = (void *)&c->context.c; const unsigned char *abuf = abuf_arg; u64 n = c->u_mode.ocb.aad_nblocks; + aesni_prepare_2_6_variable; aesni_prepare (); + aesni_prepare_2_6 (); /* Preload Offset and Sum */ asm volatile ("movdqu %[iv], %%xmm5\n\t" -- cgit v1.2.1