CoCalc -- sm3-neon-core.S

GitHub Repository: torvalds/linux
Path: blob/master/arch/arm64/crypto/sm3-neon-core.S
²⁶⁴⁵⁰ views
1
// SPDX-License-Identifier: GPL-2.0-or-later
2
/*
3
 * sm3-neon-core.S - SM3 secure hash using NEON instructions
4
 *
5
 * Linux/arm64 port of the libgcrypt SM3 implementation for AArch64
6
 *
7
 * Copyright (C) 2021 Jussi Kivilinna <[email protected]>
8
 * Copyright (c) 2022 Tianjia Zhang <[email protected]>
9
 */
10

11
#include <linux/linkage.h>
12
#include <linux/cfi_types.h>
13
#include <asm/assembler.h>
14

15
/* Context structure */
16

17
#define state_h0 0
18
#define state_h1 4
19
#define state_h2 8
20
#define state_h3 12
21
#define state_h4 16
22
#define state_h5 20
23
#define state_h6 24
24
#define state_h7 28
25

26
/* Stack structure */
27

28
#define STACK_W_SIZE        (32 * 2 * 3)
29

30
#define STACK_W             (0)
31
#define STACK_SIZE          (STACK_W + STACK_W_SIZE)
32

33
/* Register macros */
34

35
#define RSTATE x0
36
#define RDATA  x1
37
#define RNBLKS x2
38
#define RKPTR  x28
39
#define RFRAME x29
40

41
#define ra w3
42
#define rb w4
43
#define rc w5
44
#define rd w6
45
#define re w7
46
#define rf w8
47
#define rg w9
48
#define rh w10
49

50
#define t0 w11
51
#define t1 w12
52
#define t2 w13
53
#define t3 w14
54
#define t4 w15
55
#define t5 w16
56
#define t6 w17
57

58
#define k_even w19
59
#define k_odd w20
60

61
#define addr0 x21
62
#define addr1 x22
63

64
#define s0 w23
65
#define s1 w24
66
#define s2 w25
67
#define s3 w26
68

69
#define W0 v0
70
#define W1 v1
71
#define W2 v2
72
#define W3 v3
73
#define W4 v4
74
#define W5 v5
75

76
#define XTMP0 v6
77
#define XTMP1 v7
78
#define XTMP2 v16
79
#define XTMP3 v17
80
#define XTMP4 v18
81
#define XTMP5 v19
82
#define XTMP6 v20
83

84
/* Helper macros. */
85

86
#define _(...) /*_*/
87

88
#define clear_vec(x) \
89
	movi	x.8h, #0;
90

91
#define rolw(o, a, n) \
92
	ror	o, a, #(32 - n);
93

94
/* Round function macros. */
95

96
#define GG1_1(x, y, z, o, t) \
97
	eor	o, x, y;
98
#define GG1_2(x, y, z, o, t) \
99
	eor	o, o, z;
100
#define GG1_3(x, y, z, o, t)
101

102
#define FF1_1(x, y, z, o, t) GG1_1(x, y, z, o, t)
103
#define FF1_2(x, y, z, o, t)
104
#define FF1_3(x, y, z, o, t) GG1_2(x, y, z, o, t)
105

106
#define GG2_1(x, y, z, o, t) \
107
	bic	o, z, x;
108
#define GG2_2(x, y, z, o, t) \
109
	and	t, y, x;
110
#define GG2_3(x, y, z, o, t) \
111
	eor	o, o, t;
112

113
#define FF2_1(x, y, z, o, t) \
114
	eor	o, x, y;
115
#define FF2_2(x, y, z, o, t) \
116
	and	t, x, y; \
117
	and	o, o, z;
118
#define FF2_3(x, y, z, o, t) \
119
	eor	o, o, t;
120

121
#define R(i, a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \
122
	K_LOAD(round);                                                        \
123
	ldr	t5, [sp, #(wtype##_W1_ADDR(round, widx))];                    \
124
	rolw(t0, a, 12);                              /* rol(a, 12) => t0 */  \
125
      IOP(1, iop_param);                                                      \
126
	FF##i##_1(a, b, c, t1, t2);                                           \
127
	ldr	t6, [sp, #(wtype##_W1W2_ADDR(round, widx))];                  \
128
	add	k, k, e;                                                      \
129
      IOP(2, iop_param);                                                      \
130
	GG##i##_1(e, f, g, t3, t4);                                           \
131
	FF##i##_2(a, b, c, t1, t2);                                           \
132
      IOP(3, iop_param);                                                      \
133
	add	k, k, t0;                                                     \
134
	add	h, h, t5;                                                     \
135
	add	d, d, t6;                     /* w1w2 + d => d */             \
136
      IOP(4, iop_param);                                                      \
137
	rolw(k, k, 7);                        /* rol (t0 + e + t), 7) => k */ \
138
	GG##i##_2(e, f, g, t3, t4);                                           \
139
	add	h, h, k;                      /* h + w1 + k => h */           \
140
      IOP(5, iop_param);                                                      \
141
	FF##i##_3(a, b, c, t1, t2);                                           \
142
	eor	t0, t0, k;                    /* k ^ t0 => t0 */              \
143
	GG##i##_3(e, f, g, t3, t4);                                           \
144
	add	d, d, t1;                     /* FF(a,b,c) + d => d */        \
145
      IOP(6, iop_param);                                                      \
146
	add	t3, t3, h;                    /* GG(e,f,g) + h => t3 */       \
147
	rolw(b, b, 9);                        /* rol(b, 9) => b */            \
148
	eor	h, t3, t3, ror #(32-9);                                       \
149
      IOP(7, iop_param);                                                      \
150
	add	d, d, t0;                     /* t0 + d => d */               \
151
	rolw(f, f, 19);                       /* rol(f, 19) => f */           \
152
      IOP(8, iop_param);                                                      \
153
	eor	h, h, t3, ror #(32-17);       /* P0(t3) => h */
154

155
#define R1(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \
156
	R(1, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param)
157

158
#define R2(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \
159
	R(2, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param)
160

161
#define KL(round) \
162
	ldp	k_even, k_odd, [RKPTR, #(4*(round))];
163

164
/* Input expansion macros. */
165

166
/* Byte-swapped input address. */
167
#define IW_W_ADDR(round, widx, offs) \
168
	(STACK_W + ((round) / 4) * 64 + (offs) + ((widx) * 4))
169

170
/* Expanded input address. */
171
#define XW_W_ADDR(round, widx, offs) \
172
	(STACK_W + ((((round) / 3) - 4) % 2) * 64 + (offs) + ((widx) * 4))
173

174
/* Rounds 1-12, byte-swapped input block addresses. */
175
#define IW_W1_ADDR(round, widx)   IW_W_ADDR(round, widx, 32)
176
#define IW_W1W2_ADDR(round, widx) IW_W_ADDR(round, widx, 48)
177

178
/* Rounds 1-12, expanded input block addresses. */
179
#define XW_W1_ADDR(round, widx)   XW_W_ADDR(round, widx, 0)
180
#define XW_W1W2_ADDR(round, widx) XW_W_ADDR(round, widx, 16)
181

182
/* Input block loading.
183
 * Interleaving within round function needed for in-order CPUs. */
184
#define LOAD_W_VEC_1_1() \
185
	add	addr0, sp, #IW_W1_ADDR(0, 0);
186
#define LOAD_W_VEC_1_2() \
187
	add	addr1, sp, #IW_W1_ADDR(4, 0);
188
#define LOAD_W_VEC_1_3() \
189
	ld1	{W0.16b}, [RDATA], #16;
190
#define LOAD_W_VEC_1_4() \
191
	ld1	{W1.16b}, [RDATA], #16;
192
#define LOAD_W_VEC_1_5() \
193
	ld1	{W2.16b}, [RDATA], #16;
194
#define LOAD_W_VEC_1_6() \
195
	ld1	{W3.16b}, [RDATA], #16;
196
#define LOAD_W_VEC_1_7() \
197
	rev32	XTMP0.16b, W0.16b;
198
#define LOAD_W_VEC_1_8() \
199
	rev32	XTMP1.16b, W1.16b;
200
#define LOAD_W_VEC_2_1() \
201
	rev32	XTMP2.16b, W2.16b;
202
#define LOAD_W_VEC_2_2() \
203
	rev32	XTMP3.16b, W3.16b;
204
#define LOAD_W_VEC_2_3() \
205
	eor	XTMP4.16b, XTMP1.16b, XTMP0.16b;
206
#define LOAD_W_VEC_2_4() \
207
	eor	XTMP5.16b, XTMP2.16b, XTMP1.16b;
208
#define LOAD_W_VEC_2_5() \
209
	st1	{XTMP0.16b}, [addr0], #16;
210
#define LOAD_W_VEC_2_6() \
211
	st1	{XTMP4.16b}, [addr0]; \
212
	add	addr0, sp, #IW_W1_ADDR(8, 0);
213
#define LOAD_W_VEC_2_7() \
214
	eor	XTMP6.16b, XTMP3.16b, XTMP2.16b;
215
#define LOAD_W_VEC_2_8() \
216
	ext	W0.16b, XTMP0.16b, XTMP0.16b, #8;  /* W0: xx, w0, xx, xx */
217
#define LOAD_W_VEC_3_1() \
218
	mov	W2.16b, XTMP1.16b;                 /* W2: xx, w6, w5, w4 */
219
#define LOAD_W_VEC_3_2() \
220
	st1	{XTMP1.16b}, [addr1], #16;
221
#define LOAD_W_VEC_3_3() \
222
	st1	{XTMP5.16b}, [addr1]; \
223
	ext	W1.16b, XTMP0.16b, XTMP0.16b, #4;  /* W1: xx, w3, w2, w1 */
224
#define LOAD_W_VEC_3_4() \
225
	ext	W3.16b, XTMP1.16b, XTMP2.16b, #12; /* W3: xx, w9, w8, w7 */
226
#define LOAD_W_VEC_3_5() \
227
	ext	W4.16b, XTMP2.16b, XTMP3.16b, #8;  /* W4: xx, w12, w11, w10 */
228
#define LOAD_W_VEC_3_6() \
229
	st1	{XTMP2.16b}, [addr0], #16;
230
#define LOAD_W_VEC_3_7() \
231
	st1	{XTMP6.16b}, [addr0];
232
#define LOAD_W_VEC_3_8() \
233
	ext	W5.16b, XTMP3.16b, XTMP3.16b, #4;  /* W5: xx, w15, w14, w13 */
234

235
#define LOAD_W_VEC_1(iop_num, ...) \
236
	LOAD_W_VEC_1_##iop_num()
237
#define LOAD_W_VEC_2(iop_num, ...) \
238
	LOAD_W_VEC_2_##iop_num()
239
#define LOAD_W_VEC_3(iop_num, ...) \
240
	LOAD_W_VEC_3_##iop_num()
241

242
/* Message scheduling. Note: 3 words per vector register.
243
 * Interleaving within round function needed for in-order CPUs. */
244
#define SCHED_W_1_1(round, w0, w1, w2, w3, w4, w5) \
245
	/* Load (w[i - 16]) => XTMP0 */            \
246
	/* Load (w[i - 13]) => XTMP5 */            \
247
	ext	XTMP0.16b, w0.16b, w0.16b, #12;    /* XTMP0: w0, xx, xx, xx */
248
#define SCHED_W_1_2(round, w0, w1, w2, w3, w4, w5) \
249
	ext	XTMP5.16b, w1.16b, w1.16b, #12;
250
#define SCHED_W_1_3(round, w0, w1, w2, w3, w4, w5) \
251
	ext	XTMP0.16b, XTMP0.16b, w1.16b, #12; /* XTMP0: xx, w2, w1, w0 */
252
#define SCHED_W_1_4(round, w0, w1, w2, w3, w4, w5) \
253
	ext	XTMP5.16b, XTMP5.16b, w2.16b, #12;
254
#define SCHED_W_1_5(round, w0, w1, w2, w3, w4, w5) \
255
	/* w[i - 9] == w3 */                       \
256
	/* W3 ^ XTMP0 => XTMP0 */                  \
257
	eor	XTMP0.16b, XTMP0.16b, w3.16b;
258
#define SCHED_W_1_6(round, w0, w1, w2, w3, w4, w5) \
259
	/* w[i - 3] == w5 */                       \
260
	/* rol(XMM5, 15) ^ XTMP0 => XTMP0 */       \
261
	/* rol(XTMP5, 7) => XTMP1 */               \
262
	add	addr0, sp, #XW_W1_ADDR((round), 0); \
263
	shl	XTMP2.4s, w5.4s, #15;
264
#define SCHED_W_1_7(round, w0, w1, w2, w3, w4, w5) \
265
	shl	XTMP1.4s, XTMP5.4s, #7;
266
#define SCHED_W_1_8(round, w0, w1, w2, w3, w4, w5) \
267
	sri	XTMP2.4s, w5.4s, #(32-15);
268
#define SCHED_W_2_1(round, w0, w1, w2, w3, w4, w5) \
269
	sri	XTMP1.4s, XTMP5.4s, #(32-7);
270
#define SCHED_W_2_2(round, w0, w1, w2, w3, w4, w5) \
271
	eor	XTMP0.16b, XTMP0.16b, XTMP2.16b;
272
#define SCHED_W_2_3(round, w0, w1, w2, w3, w4, w5) \
273
	/* w[i - 6] == W4 */                       \
274
	/* W4 ^ XTMP1 => XTMP1 */                  \
275
	eor	XTMP1.16b, XTMP1.16b, w4.16b;
276
#define SCHED_W_2_4(round, w0, w1, w2, w3, w4, w5) \
277
	/* P1(XTMP0) ^ XTMP1 => W0 */              \
278
	shl	XTMP3.4s, XTMP0.4s, #15;
279
#define SCHED_W_2_5(round, w0, w1, w2, w3, w4, w5) \
280
	shl	XTMP4.4s, XTMP0.4s, #23;
281
#define SCHED_W_2_6(round, w0, w1, w2, w3, w4, w5) \
282
	eor	w0.16b, XTMP1.16b, XTMP0.16b;
283
#define SCHED_W_2_7(round, w0, w1, w2, w3, w4, w5) \
284
	sri	XTMP3.4s, XTMP0.4s, #(32-15);
285
#define SCHED_W_2_8(round, w0, w1, w2, w3, w4, w5) \
286
	sri	XTMP4.4s, XTMP0.4s, #(32-23);
287
#define SCHED_W_3_1(round, w0, w1, w2, w3, w4, w5) \
288
	eor	w0.16b, w0.16b, XTMP3.16b;
289
#define SCHED_W_3_2(round, w0, w1, w2, w3, w4, w5) \
290
	/* Load (w[i - 3]) => XTMP2 */             \
291
	ext	XTMP2.16b, w4.16b, w4.16b, #12;
292
#define SCHED_W_3_3(round, w0, w1, w2, w3, w4, w5) \
293
	eor	w0.16b, w0.16b, XTMP4.16b;
294
#define SCHED_W_3_4(round, w0, w1, w2, w3, w4, w5) \
295
	ext	XTMP2.16b, XTMP2.16b, w5.16b, #12;
296
#define SCHED_W_3_5(round, w0, w1, w2, w3, w4, w5) \
297
	/* W1 ^ W2 => XTMP3 */                     \
298
	eor	XTMP3.16b, XTMP2.16b, w0.16b;
299
#define SCHED_W_3_6(round, w0, w1, w2, w3, w4, w5)
300
#define SCHED_W_3_7(round, w0, w1, w2, w3, w4, w5) \
301
	st1	{XTMP2.16b-XTMP3.16b}, [addr0];
302
#define SCHED_W_3_8(round, w0, w1, w2, w3, w4, w5)
303

304
#define SCHED_W_W0W1W2W3W4W5_1(iop_num, round) \
305
	SCHED_W_1_##iop_num(round, W0, W1, W2, W3, W4, W5)
306
#define SCHED_W_W0W1W2W3W4W5_2(iop_num, round) \
307
	SCHED_W_2_##iop_num(round, W0, W1, W2, W3, W4, W5)
308
#define SCHED_W_W0W1W2W3W4W5_3(iop_num, round) \
309
	SCHED_W_3_##iop_num(round, W0, W1, W2, W3, W4, W5)
310

311
#define SCHED_W_W1W2W3W4W5W0_1(iop_num, round) \
312
	SCHED_W_1_##iop_num(round, W1, W2, W3, W4, W5, W0)
313
#define SCHED_W_W1W2W3W4W5W0_2(iop_num, round) \
314
	SCHED_W_2_##iop_num(round, W1, W2, W3, W4, W5, W0)
315
#define SCHED_W_W1W2W3W4W5W0_3(iop_num, round) \
316
	SCHED_W_3_##iop_num(round, W1, W2, W3, W4, W5, W0)
317

318
#define SCHED_W_W2W3W4W5W0W1_1(iop_num, round) \
319
	SCHED_W_1_##iop_num(round, W2, W3, W4, W5, W0, W1)
320
#define SCHED_W_W2W3W4W5W0W1_2(iop_num, round) \
321
	SCHED_W_2_##iop_num(round, W2, W3, W4, W5, W0, W1)
322
#define SCHED_W_W2W3W4W5W0W1_3(iop_num, round) \
323
	SCHED_W_3_##iop_num(round, W2, W3, W4, W5, W0, W1)
324

325
#define SCHED_W_W3W4W5W0W1W2_1(iop_num, round) \
326
	SCHED_W_1_##iop_num(round, W3, W4, W5, W0, W1, W2)
327
#define SCHED_W_W3W4W5W0W1W2_2(iop_num, round) \
328
	SCHED_W_2_##iop_num(round, W3, W4, W5, W0, W1, W2)
329
#define SCHED_W_W3W4W5W0W1W2_3(iop_num, round) \
330
	SCHED_W_3_##iop_num(round, W3, W4, W5, W0, W1, W2)
331

332
#define SCHED_W_W4W5W0W1W2W3_1(iop_num, round) \
333
	SCHED_W_1_##iop_num(round, W4, W5, W0, W1, W2, W3)
334
#define SCHED_W_W4W5W0W1W2W3_2(iop_num, round) \
335
	SCHED_W_2_##iop_num(round, W4, W5, W0, W1, W2, W3)
336
#define SCHED_W_W4W5W0W1W2W3_3(iop_num, round) \
337
	SCHED_W_3_##iop_num(round, W4, W5, W0, W1, W2, W3)
338

339
#define SCHED_W_W5W0W1W2W3W4_1(iop_num, round) \
340
	SCHED_W_1_##iop_num(round, W5, W0, W1, W2, W3, W4)
341
#define SCHED_W_W5W0W1W2W3W4_2(iop_num, round) \
342
	SCHED_W_2_##iop_num(round, W5, W0, W1, W2, W3, W4)
343
#define SCHED_W_W5W0W1W2W3W4_3(iop_num, round) \
344
	SCHED_W_3_##iop_num(round, W5, W0, W1, W2, W3, W4)
345

346

347
	/*
348
	 * Transform blocks*64 bytes (blocks*16 32-bit words) at 'src'.
349
	 *
350
	 * void sm3_neon_transform(struct sm3_state *sst, u8 const *src,
351
	 *                         int blocks)
352
	 */
353
	.text
354
.align 3
355
SYM_TYPED_FUNC_START(sm3_neon_transform)
356
	ldp		ra, rb, [RSTATE, #0]
357
	ldp		rc, rd, [RSTATE, #8]
358
	ldp		re, rf, [RSTATE, #16]
359
	ldp		rg, rh, [RSTATE, #24]
360

361
	stp		x28, x29, [sp, #-16]!
362
	stp		x19, x20, [sp, #-16]!
363
	stp		x21, x22, [sp, #-16]!
364
	stp		x23, x24, [sp, #-16]!
365
	stp		x25, x26, [sp, #-16]!
366
	mov		RFRAME, sp
367

368
	sub		addr0, sp, #STACK_SIZE
369
	adr_l		RKPTR, .LKtable
370
	and		sp, addr0, #(~63)
371

372
	/* Preload first block. */
373
	LOAD_W_VEC_1(1, 0)
374
	LOAD_W_VEC_1(2, 0)
375
	LOAD_W_VEC_1(3, 0)
376
	LOAD_W_VEC_1(4, 0)
377
	LOAD_W_VEC_1(5, 0)
378
	LOAD_W_VEC_1(6, 0)
379
	LOAD_W_VEC_1(7, 0)
380
	LOAD_W_VEC_1(8, 0)
381
	LOAD_W_VEC_2(1, 0)
382
	LOAD_W_VEC_2(2, 0)
383
	LOAD_W_VEC_2(3, 0)
384
	LOAD_W_VEC_2(4, 0)
385
	LOAD_W_VEC_2(5, 0)
386
	LOAD_W_VEC_2(6, 0)
387
	LOAD_W_VEC_2(7, 0)
388
	LOAD_W_VEC_2(8, 0)
389
	LOAD_W_VEC_3(1, 0)
390
	LOAD_W_VEC_3(2, 0)
391
	LOAD_W_VEC_3(3, 0)
392
	LOAD_W_VEC_3(4, 0)
393
	LOAD_W_VEC_3(5, 0)
394
	LOAD_W_VEC_3(6, 0)
395
	LOAD_W_VEC_3(7, 0)
396
	LOAD_W_VEC_3(8, 0)
397

398
.balign 16
399
.Loop:
400
	/* Transform 0-3 */
401
	R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 0, 0, IW, _, 0)
402
	R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  1, 1, IW, _, 0)
403
	R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 2, 2, IW, _, 0)
404
	R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  3, 3, IW, _, 0)
405

406
	/* Transform 4-7 + Precalc 12-14 */
407
	R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 4, 0, IW, _, 0)
408
	R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  5, 1, IW, _, 0)
409
	R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 6, 2, IW, SCHED_W_W0W1W2W3W4W5_1, 12)
410
	R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  7, 3, IW, SCHED_W_W0W1W2W3W4W5_2, 12)
411

412
	/* Transform 8-11 + Precalc 12-17 */
413
	R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 8, 0, IW, SCHED_W_W0W1W2W3W4W5_3, 12)
414
	R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  9, 1, IW, SCHED_W_W1W2W3W4W5W0_1, 15)
415
	R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 10, 2, IW, SCHED_W_W1W2W3W4W5W0_2, 15)
416
	R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  11, 3, IW, SCHED_W_W1W2W3W4W5W0_3, 15)
417

418
	/* Transform 12-14 + Precalc 18-20 */
419
	R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 12, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 18)
420
	R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  13, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 18)
421
	R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 14, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 18)
422

423
	/* Transform 15-17 + Precalc 21-23 */
424
	R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  15, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 21)
425
	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 16, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 21)
426
	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  17, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 21)
427

428
	/* Transform 18-20 + Precalc 24-26 */
429
	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 18, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 24)
430
	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  19, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 24)
431
	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 20, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 24)
432

433
	/* Transform 21-23 + Precalc 27-29 */
434
	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  21, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 27)
435
	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 22, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 27)
436
	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  23, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 27)
437

438
	/* Transform 24-26 + Precalc 30-32 */
439
	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 24, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 30)
440
	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  25, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 30)
441
	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 26, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 30)
442

443
	/* Transform 27-29 + Precalc 33-35 */
444
	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  27, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 33)
445
	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 28, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 33)
446
	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  29, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 33)
447

448
	/* Transform 30-32 + Precalc 36-38 */
449
	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 30, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 36)
450
	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  31, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 36)
451
	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 32, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 36)
452

453
	/* Transform 33-35 + Precalc 39-41 */
454
	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  33, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 39)
455
	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 34, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 39)
456
	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  35, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 39)
457

458
	/* Transform 36-38 + Precalc 42-44 */
459
	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 36, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 42)
460
	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  37, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 42)
461
	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 38, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 42)
462

463
	/* Transform 39-41 + Precalc 45-47 */
464
	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  39, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 45)
465
	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 40, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 45)
466
	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  41, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 45)
467

468
	/* Transform 42-44 + Precalc 48-50 */
469
	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 42, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 48)
470
	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  43, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 48)
471
	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 44, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 48)
472

473
	/* Transform 45-47 + Precalc 51-53 */
474
	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  45, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 51)
475
	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 46, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 51)
476
	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  47, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 51)
477

478
	/* Transform 48-50 + Precalc 54-56 */
479
	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 48, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 54)
480
	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  49, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 54)
481
	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 50, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 54)
482

483
	/* Transform 51-53 + Precalc 57-59 */
484
	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  51, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 57)
485
	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 52, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 57)
486
	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  53, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 57)
487

488
	/* Transform 54-56 + Precalc 60-62 */
489
	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 54, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 60)
490
	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  55, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 60)
491
	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 56, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 60)
492

493
	/* Transform 57-59 + Precalc 63 */
494
	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  57, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 63)
495
	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 58, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 63)
496
	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  59, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 63)
497

498
	/* Transform 60 */
499
	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 60, 0, XW, _, _)
500
	subs		RNBLKS, RNBLKS, #1
501
	b.eq		.Lend
502

503
	/* Transform 61-63 + Preload next block */
504
	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  61, 1, XW, LOAD_W_VEC_1, _)
505
	ldp		s0, s1, [RSTATE, #0]
506
	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, LOAD_W_VEC_2, _)
507
	ldp		s2, s3, [RSTATE, #8]
508
	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  63, 0, XW, LOAD_W_VEC_3, _)
509

510
	/* Update the chaining variables. */
511
	eor		ra, ra, s0
512
	eor		rb, rb, s1
513
	ldp		s0, s1, [RSTATE, #16]
514
	eor		rc, rc, s2
515
	ldp		k_even, k_odd, [RSTATE, #24]
516
	eor		rd, rd, s3
517
	eor		re, re, s0
518
	stp		ra, rb, [RSTATE, #0]
519
	eor		rf, rf, s1
520
	stp		rc, rd, [RSTATE, #8]
521
	eor		rg, rg, k_even
522
	stp		re, rf, [RSTATE, #16]
523
	eor		rh, rh, k_odd
524
	stp		rg, rh, [RSTATE, #24]
525
	b		.Loop
526

527
.Lend:
528
	/* Transform 61-63 */
529
	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  61, 1, XW, _, _)
530
	ldp		s0, s1, [RSTATE, #0]
531
	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, _, _)
532
	ldp		s2, s3, [RSTATE, #8]
533
	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  63, 0, XW, _, _)
534

535
	/* Update the chaining variables. */
536
	eor		ra, ra, s0
537
	clear_vec(W0)
538
	eor		rb, rb, s1
539
	clear_vec(W1)
540
	ldp		s0, s1, [RSTATE, #16]
541
	clear_vec(W2)
542
	eor		rc, rc, s2
543
	clear_vec(W3)
544
	ldp		k_even, k_odd, [RSTATE, #24]
545
	clear_vec(W4)
546
	eor		rd, rd, s3
547
	clear_vec(W5)
548
	eor		re, re, s0
549
	clear_vec(XTMP0)
550
	stp		ra, rb, [RSTATE, #0]
551
	clear_vec(XTMP1)
552
	eor		rf, rf, s1
553
	clear_vec(XTMP2)
554
	stp		rc, rd, [RSTATE, #8]
555
	clear_vec(XTMP3)
556
	eor		rg, rg, k_even
557
	clear_vec(XTMP4)
558
	stp		re, rf, [RSTATE, #16]
559
	clear_vec(XTMP5)
560
	eor		rh, rh, k_odd
561
	clear_vec(XTMP6)
562
	stp		rg, rh, [RSTATE, #24]
563

564
	/* Clear message expansion area */
565
	add		addr0, sp, #STACK_W
566
	st1		{W0.16b-W3.16b}, [addr0], #64
567
	st1		{W0.16b-W3.16b}, [addr0], #64
568
	st1		{W0.16b-W3.16b}, [addr0]
569

570
	mov		sp, RFRAME
571

572
	ldp		x25, x26, [sp], #16
573
	ldp		x23, x24, [sp], #16
574
	ldp		x21, x22, [sp], #16
575
	ldp		x19, x20, [sp], #16
576
	ldp		x28, x29, [sp], #16
577

578
	ret
579
SYM_FUNC_END(sm3_neon_transform)
580

581

582
	.section	".rodata", "a"
583

584
	.align 4
585
.LKtable:
586
	.long 0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb
587
	.long 0x9cc45197, 0x3988a32f, 0x7311465e, 0xe6228cbc
588
	.long 0xcc451979, 0x988a32f3, 0x311465e7, 0x6228cbce
589
	.long 0xc451979c, 0x88a32f39, 0x11465e73, 0x228cbce6
590
	.long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c
591
	.long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce
592
	.long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec
593
	.long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5
594
	.long 0x7a879d8a, 0xf50f3b14, 0xea1e7629, 0xd43cec53
595
	.long 0xa879d8a7, 0x50f3b14f, 0xa1e7629e, 0x43cec53d
596
	.long 0x879d8a7a, 0x0f3b14f5, 0x1e7629ea, 0x3cec53d4
597
	.long 0x79d8a7a8, 0xf3b14f50, 0xe7629ea1, 0xcec53d43
598
	.long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c
599
	.long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce
600
	.long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec
601
	.long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5
602

603
Product

Resources

Company
