1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
|
/* whirlpool-sse2-amd64.S - AMD64 assembly implementation of Whirlpool
*
* Copyright (C) 2014 Jussi Kivilinna <jussi.kivilinna@iki.fi>
*
* This file is part of Libgcrypt.
*
* Libgcrypt is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of
* the License, or (at your option) any later version.
*
* Libgcrypt is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#ifdef __x86_64
#include <config.h>
#if (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \
defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS)) && defined(USE_WHIRLPOOL)
#ifdef __PIC__
# define RIP %rip
#else
# define RIP
#endif
#ifdef HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS
# define ELF(...) __VA_ARGS__
#else
# define ELF(...) /*_*/
#endif
.text
/* look-up table offsets on RTAB */
#define RC (0)
#define C0 (RC + (8 * 10))
#define C1 (C0 + (8 * 256))
#define C2 (C1 + (8 * 256))
#define C3 (C2 + (8 * 256))
#define C4 (C3 + (8 * 256))
#define C5 (C4 + (8 * 256))
#define C6 (C5 + (8 * 256))
#define C7 (C6 + (8 * 256))
/* stack variables */
#define STACK_DATAP (0)
#define STACK_STATEP (STACK_DATAP + 8)
#define STACK_ROUNDS (STACK_STATEP + 8)
#define STACK_NBLKS (STACK_ROUNDS + 8)
#define STACK_RBP (STACK_NBLKS + 8)
#define STACK_RBX (STACK_RBP + 8)
#define STACK_R12 (STACK_RBX + 8)
#define STACK_R13 (STACK_R12 + 8)
#define STACK_R14 (STACK_R13 + 8)
#define STACK_R15 (STACK_R14 + 8)
#define STACK_MAX (STACK_R15 + 8)
/* register macros */
#define RTAB %rbp
#define RI1 %rax
#define RI2 %rbx
#define RI3 %rcx
#define RI4 %rdx
#define RI1d %eax
#define RI2d %ebx
#define RI3d %ecx
#define RI4d %edx
#define RI1bl %al
#define RI2bl %bl
#define RI3bl %cl
#define RI4bl %dl
#define RI1bh %ah
#define RI2bh %bh
#define RI3bh %ch
#define RI4bh %dh
#define RB0 %r8
#define RB1 %r9
#define RB2 %r10
#define RB3 %r11
#define RB4 %r12
#define RB5 %r13
#define RB6 %r14
#define RB7 %r15
#define RT0 %rsi
#define RT1 %rdi
#define RT0d %esi
#define RT1d %edi
#define XKEY0 %xmm0
#define XKEY1 %xmm1
#define XKEY2 %xmm2
#define XKEY3 %xmm3
#define XKEY4 %xmm4
#define XKEY5 %xmm5
#define XKEY6 %xmm6
#define XKEY7 %xmm7
#define XSTATE0 %xmm8
#define XSTATE1 %xmm9
#define XSTATE2 %xmm10
#define XSTATE3 %xmm11
#define XSTATE4 %xmm12
#define XSTATE5 %xmm13
#define XSTATE6 %xmm14
#define XSTATE7 %xmm15
/***********************************************************************
* AMD64 assembly implementation of Whirlpool.
* - Using table-lookups
* - Store state in XMM registers
***********************************************************************/
#define __do_whirl(op, ri, \
b0, b1, b2, b3, b4, b5, b6, b7, \
load_ri, load_arg) \
movzbl ri ## bl, RT0d; \
movzbl ri ## bh, RT1d; \
shrq $16, ri; \
op ## q C7(RTAB,RT0,8), b7; \
op ## q C6(RTAB,RT1,8), b6; \
movzbl ri ## bl, RT0d; \
movzbl ri ## bh, RT1d; \
shrq $16, ri; \
op ## q C5(RTAB,RT0,8), b5; \
op ## q C4(RTAB,RT1,8), b4; \
movzbl ri ## bl, RT0d; \
movzbl ri ## bh, RT1d; \
shrl $16, ri ## d; \
op ## q C3(RTAB,RT0,8), b3; \
op ## q C2(RTAB,RT1,8), b2; \
movzbl ri ## bl, RT0d; \
movzbl ri ## bh, RT1d; \
load_ri( load_arg, ri); \
op ## q C1(RTAB,RT0,8), b1; \
op ## q C0(RTAB,RT1,8), b0;
#define do_whirl(op, ri, rb_add, load_ri, load_arg) \
__do_whirl(op, ##ri, rb_add, load_ri, load_arg)
#define dummy(...) /*_*/
#define do_movq(src, dst) movq src, dst;
#define RB_ADD0 RB0, RB1, RB2, RB3, RB4, RB5, RB6, RB7
#define RB_ADD1 RB1, RB2, RB3, RB4, RB5, RB6, RB7, RB0
#define RB_ADD2 RB2, RB3, RB4, RB5, RB6, RB7, RB0, RB1
#define RB_ADD3 RB3, RB4, RB5, RB6, RB7, RB0, RB1, RB2
#define RB_ADD4 RB4, RB5, RB6, RB7, RB0, RB1, RB2, RB3
#define RB_ADD5 RB5, RB6, RB7, RB0, RB1, RB2, RB3, RB4
#define RB_ADD6 RB6, RB7, RB0, RB1, RB2, RB3, RB4, RB5
#define RB_ADD7 RB7, RB0, RB1, RB2, RB3, RB4, RB5, RB6
.align 8
.globl _gcry_whirlpool_transform_amd64
ELF(.type _gcry_whirlpool_transform_amd64,@function;)
_gcry_whirlpool_transform_amd64:
/* input:
* %rdi: state
* %rsi: inblk
* %rdx: nblks
* %rcx: look-up tables
*/
cmp $0, %rdx;
je .Lskip;
subq $STACK_MAX, %rsp;
movq %rbp, STACK_RBP(%rsp);
movq %rbx, STACK_RBX(%rsp);
movq %r12, STACK_R12(%rsp);
movq %r13, STACK_R13(%rsp);
movq %r14, STACK_R14(%rsp);
movq %r15, STACK_R15(%rsp);
movq %rdx, STACK_NBLKS(%rsp);
movq %rdi, STACK_STATEP(%rsp);
movq %rsi, STACK_DATAP(%rsp);
movq %rcx, RTAB;
jmp .Lfirst_block;
.align 8
.Lblock_loop:
movq STACK_DATAP(%rsp), %rsi;
movq RI1, %rdi;
.Lfirst_block:
/* load data_block */
movq 0*8(%rsi), RB0;
movq 1*8(%rsi), RB1;
bswapq RB0;
movq 2*8(%rsi), RB2;
bswapq RB1;
movq 3*8(%rsi), RB3;
bswapq RB2;
movq 4*8(%rsi), RB4;
bswapq RB3;
movq 5*8(%rsi), RB5;
bswapq RB4;
movq RB0, XSTATE0;
movq 6*8(%rsi), RB6;
bswapq RB5;
movq RB1, XSTATE1;
movq 7*8(%rsi), RB7;
bswapq RB6;
movq RB2, XSTATE2;
bswapq RB7;
movq RB3, XSTATE3;
movq RB4, XSTATE4;
movq RB5, XSTATE5;
movq RB6, XSTATE6;
movq RB7, XSTATE7;
/* load key */
movq 0*8(%rdi), XKEY0;
movq 1*8(%rdi), XKEY1;
movq 2*8(%rdi), XKEY2;
movq 3*8(%rdi), XKEY3;
movq 4*8(%rdi), XKEY4;
movq 5*8(%rdi), XKEY5;
movq 6*8(%rdi), XKEY6;
movq 7*8(%rdi), XKEY7;
movq XKEY0, RI1;
movq XKEY1, RI2;
movq XKEY2, RI3;
movq XKEY3, RI4;
/* prepare and store state */
pxor XKEY0, XSTATE0;
pxor XKEY1, XSTATE1;
pxor XKEY2, XSTATE2;
pxor XKEY3, XSTATE3;
pxor XKEY4, XSTATE4;
pxor XKEY5, XSTATE5;
pxor XKEY6, XSTATE6;
pxor XKEY7, XSTATE7;
movq XSTATE0, 0*8(%rdi);
movq XSTATE1, 1*8(%rdi);
movq XSTATE2, 2*8(%rdi);
movq XSTATE3, 3*8(%rdi);
movq XSTATE4, 4*8(%rdi);
movq XSTATE5, 5*8(%rdi);
movq XSTATE6, 6*8(%rdi);
movq XSTATE7, 7*8(%rdi);
addq $64, STACK_DATAP(%rsp);
movl $(0), STACK_ROUNDS(%rsp);
.align 8
.Lround_loop:
do_whirl(mov, RI1 /*XKEY0*/, RB_ADD0, do_movq, XKEY4);
do_whirl(xor, RI2 /*XKEY1*/, RB_ADD1, do_movq, XKEY5);
do_whirl(xor, RI3 /*XKEY2*/, RB_ADD2, do_movq, XKEY6);
do_whirl(xor, RI4 /*XKEY3*/, RB_ADD3, do_movq, XKEY7);
do_whirl(xor, RI1 /*XKEY0*/, RB_ADD4, do_movq, XSTATE0);
do_whirl(xor, RI2 /*XKEY1*/, RB_ADD5, do_movq, XSTATE1);
do_whirl(xor, RI3 /*XKEY2*/, RB_ADD6, do_movq, XSTATE2);
do_whirl(xor, RI4 /*XKEY3*/, RB_ADD7, do_movq, XSTATE3);
movl STACK_ROUNDS(%rsp), RT0d;
movq RB1, XKEY1;
addl $1, STACK_ROUNDS(%rsp);
movq RB2, XKEY2;
movq RB3, XKEY3;
xorq RC(RTAB,RT0,8), RB0; /* Add round constant */
movq RB4, XKEY4;
movq RB5, XKEY5;
movq RB0, XKEY0;
movq RB6, XKEY6;
movq RB7, XKEY7;
do_whirl(xor, RI1 /*XSTATE0*/, RB_ADD0, do_movq, XSTATE4);
do_whirl(xor, RI2 /*XSTATE1*/, RB_ADD1, do_movq, XSTATE5);
do_whirl(xor, RI3 /*XSTATE2*/, RB_ADD2, do_movq, XSTATE6);
do_whirl(xor, RI4 /*XSTATE3*/, RB_ADD3, do_movq, XSTATE7);
cmpl $10, STACK_ROUNDS(%rsp);
je .Lis_last_round;
do_whirl(xor, RI1 /*XSTATE4*/, RB_ADD4, do_movq, XKEY0);
do_whirl(xor, RI2 /*XSTATE5*/, RB_ADD5, do_movq, XKEY1);
do_whirl(xor, RI3 /*XSTATE6*/, RB_ADD6, do_movq, XKEY2);
do_whirl(xor, RI4 /*XSTATE7*/, RB_ADD7, do_movq, XKEY3);
movq RB0, XSTATE0;
movq RB1, XSTATE1;
movq RB2, XSTATE2;
movq RB3, XSTATE3;
movq RB4, XSTATE4;
movq RB5, XSTATE5;
movq RB6, XSTATE6;
movq RB7, XSTATE7;
jmp .Lround_loop;
.align 8
.Lis_last_round:
do_whirl(xor, RI1 /*XSTATE4*/, RB_ADD4, dummy, _);
movq STACK_STATEP(%rsp), RI1;
do_whirl(xor, RI2 /*XSTATE5*/, RB_ADD5, dummy, _);
do_whirl(xor, RI3 /*XSTATE6*/, RB_ADD6, dummy, _);
do_whirl(xor, RI4 /*XSTATE7*/, RB_ADD7, dummy, _);
/* store state */
xorq RB0, 0*8(RI1);
xorq RB1, 1*8(RI1);
xorq RB2, 2*8(RI1);
xorq RB3, 3*8(RI1);
xorq RB4, 4*8(RI1);
xorq RB5, 5*8(RI1);
xorq RB6, 6*8(RI1);
xorq RB7, 7*8(RI1);
subq $1, STACK_NBLKS(%rsp);
jnz .Lblock_loop;
movq STACK_RBP(%rsp), %rbp;
movq STACK_RBX(%rsp), %rbx;
movq STACK_R12(%rsp), %r12;
movq STACK_R13(%rsp), %r13;
movq STACK_R14(%rsp), %r14;
movq STACK_R15(%rsp), %r15;
addq $STACK_MAX, %rsp;
.Lskip:
movl $(STACK_MAX + 8), %eax;
ret;
ELF(.size _gcry_whirlpool_transform_amd64,.-_gcry_whirlpool_transform_amd64;)
#endif
#endif
|
