CoCalc -- vpsm4_ex-armv8.S

GitHub Repository: freebsd/freebsd-src
Path: blob/main/sys/crypto/openssl/aarch64/vpsm4_ex-armv8.S
¹⁰⁸⁶¹¹ views
1
/* Do not modify. This file is auto-generated from vpsm4_ex-armv8.pl. */
2
// Copyright 2022-2026 The OpenSSL Project Authors. All Rights Reserved.
3
//
4
// Licensed under the Apache License 2.0 (the "License").  You may not use
5
// this file except in compliance with the License.  You can obtain a copy
6
// in the file LICENSE in the source distribution or at
7
// https://www.openssl.org/source/license.html
8

9
//
10
// This module implements SM4 with ASIMD and AESE on AARCH64
11
//
12
// Dec 2022
13
//
14

15
// $output is the last argument if it looks like a file (it has an extension)
16
// $flavour is the first argument if it doesn't look like a file
17
#include "arm_arch.h"
18
.arch	armv8-a+crypto
19
.text
20

21
.type	_vpsm4_ex_consts,%object
22
.align	7
23
_vpsm4_ex_consts:
24
.Lck:
25
.long	0x00070E15, 0x1C232A31, 0x383F464D, 0x545B6269
26
.long	0x70777E85, 0x8C939AA1, 0xA8AFB6BD, 0xC4CBD2D9
27
.long	0xE0E7EEF5, 0xFC030A11, 0x181F262D, 0x343B4249
28
.long	0x50575E65, 0x6C737A81, 0x888F969D, 0xA4ABB2B9
29
.long	0xC0C7CED5, 0xDCE3EAF1, 0xF8FF060D, 0x141B2229
30
.long	0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299
31
.long	0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209
32
.long	0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279
33
.Lfk:
34
.quad	0x56aa3350a3b1bac6,0xb27022dc677d9197
35
.Lshuffles:
36
.quad	0x0B0A090807060504,0x030201000F0E0D0C
37
.Lxts_magic:
38
#ifndef __AARCH64EB__
39
.quad	0x0101010101010187,0x0101010101010101
40
#else
41
.quad	0x0101010101010101,0x0101010101010187
42
#endif
43
.Lsbox_magic:
44
#ifndef __AARCH64EB__
45
.quad	0x0b0e0104070a0d00,0x0306090c0f020508
46
.quad	0x62185a2042387a00,0x22581a6002783a40
47
.quad	0x15df62a89e54e923,0xc10bb67c4a803df7
48
.quad	0xb9aa6b78c1d21300,0x1407c6d56c7fbead
49
.quad	0x6404462679195b3b,0xe383c1a1fe9edcbc
50
#else
51
.quad	0x0306090c0f020508,0x0b0e0104070a0d00
52
.quad	0x22581a6002783a40,0x62185a2042387a00
53
.quad	0xc10bb67c4a803df7,0x15df62a89e54e923
54
.quad	0x1407c6d56c7fbead,0xb9aa6b78c1d21300
55
.quad	0xe383c1a1fe9edcbc,0x6404462679195b3b
56
#endif
57
.quad	0x0f0f0f0f0f0f0f0f,0x0f0f0f0f0f0f0f0f
58

59
.size	_vpsm4_ex_consts,.-_vpsm4_ex_consts
60
.type	_vpsm4_ex_set_key,%function
61
.align	4
62
_vpsm4_ex_set_key:
63
	AARCH64_VALID_CALL_TARGET
64
	ld1	{v5.4s},[x0]
65
	adrp	x9, .Lsbox_magic
66
	ldr	q26, [x9, #:lo12:.Lsbox_magic]
67
	ldr	q27, [x9, #:lo12:.Lsbox_magic+16]
68
	ldr	q28, [x9, #:lo12:.Lsbox_magic+32]
69
	ldr	q29, [x9, #:lo12:.Lsbox_magic+48]
70
	ldr	q30, [x9, #:lo12:.Lsbox_magic+64]
71
	ldr	q31, [x9, #:lo12:.Lsbox_magic+80]
72
#ifndef __AARCH64EB__
73
	rev32	v5.16b,v5.16b
74
#endif
75
	adrp	x5,.Lshuffles
76
	add	x5,x5,#:lo12:.Lshuffles
77
	ld1	{v7.2d},[x5]
78
	adrp	x5,.Lfk
79
	add	x5,x5,#:lo12:.Lfk
80
	ld1	{v6.2d},[x5]
81
	eor	v5.16b,v5.16b,v6.16b
82
	mov	x6,#32
83
	adrp	x5,.Lck
84
	add	x5,x5,#:lo12:.Lck
85
	movi	v0.16b,#64
86
	cbnz	w2,1f
87
	add	x1,x1,124
88
1:
89
	mov	w7,v5.s[1]
90
	ldr	w8,[x5],#4
91
	eor	w8,w8,w7
92
	mov	w7,v5.s[2]
93
	eor	w8,w8,w7
94
	mov	w7,v5.s[3]
95
	eor	w8,w8,w7
96
	// optimize sbox using AESE instruction
97
	mov	v4.s[0],w8
98
	tbl	v0.16b, {v4.16b}, v26.16b
99
	ushr	v2.16b, v0.16b, 4
100
	and	v0.16b, v0.16b, v31.16b
101
	tbl	v0.16b, {v28.16b}, v0.16b
102
	tbl	v2.16b, {v27.16b}, v2.16b
103
	eor	v0.16b, v0.16b, v2.16b
104
	eor	v1.16b, v1.16b, v1.16b
105
	aese	v0.16b,v1.16b
106
	ushr	v2.16b, v0.16b, 4
107
	and	v0.16b, v0.16b, v31.16b
108
	tbl	v0.16b, {v30.16b}, v0.16b
109
	tbl	v2.16b, {v29.16b}, v2.16b
110
	eor	v0.16b, v0.16b, v2.16b
111
	mov	w7,v0.s[0]
112
	eor	w8,w7,w7,ror #19
113
	eor	w8,w8,w7,ror #9
114
	mov	w7,v5.s[0]
115
	eor	w8,w8,w7
116
	mov	v5.s[0],w8
117
	cbz	w2,2f
118
	str	w8,[x1],#4
119
	b	3f
120
2:
121
	str	w8,[x1],#-4
122
3:
123
	tbl	v5.16b,{v5.16b},v7.16b
124
	subs	x6,x6,#1
125
	b.ne	1b
126
	ret
127
.size	_vpsm4_ex_set_key,.-_vpsm4_ex_set_key
128
.type	_vpsm4_ex_enc_4blks,%function
129
.align	4
130
_vpsm4_ex_enc_4blks:
131
	AARCH64_VALID_CALL_TARGET
132
	mov	x10,x3
133
	mov	w11,#8
134
10:
135
	ldp	w7,w8,[x10],8
136
	dup	v12.4s,w7
137
	dup	v13.4s,w8
138

139
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
140
	eor	v14.16b,v6.16b,v7.16b
141
	eor	v12.16b,v5.16b,v12.16b
142
	eor	v12.16b,v14.16b,v12.16b
143
	// optimize sbox using AESE instruction
144
	tbl	v0.16b, {v12.16b}, v26.16b
145
	ushr	v24.16b, v0.16b, 4
146
	and	v0.16b, v0.16b, v31.16b
147
	tbl	v0.16b, {v28.16b}, v0.16b
148
	tbl	v24.16b, {v27.16b}, v24.16b
149
	eor	v0.16b, v0.16b, v24.16b
150
	eor	v1.16b, v1.16b, v1.16b
151
	aese	v0.16b,v1.16b
152
	ushr	v24.16b, v0.16b, 4
153
	and	v0.16b, v0.16b, v31.16b
154
	tbl	v0.16b, {v30.16b}, v0.16b
155
	tbl	v24.16b, {v29.16b}, v24.16b
156
	eor	v0.16b, v0.16b, v24.16b
157
	mov	v12.16b,v0.16b
158

159
	// linear transformation
160
	ushr	v0.4s,v12.4s,32-2
161
	ushr	v1.4s,v12.4s,32-10
162
	ushr	v2.4s,v12.4s,32-18
163
	ushr	v3.4s,v12.4s,32-24
164
	sli	v0.4s,v12.4s,2
165
	sli	v1.4s,v12.4s,10
166
	sli	v2.4s,v12.4s,18
167
	sli	v3.4s,v12.4s,24
168
	eor	v24.16b,v0.16b,v12.16b
169
	eor	v24.16b,v24.16b,v1.16b
170
	eor	v12.16b,v2.16b,v3.16b
171
	eor	v12.16b,v12.16b,v24.16b
172
	eor	v4.16b,v4.16b,v12.16b
173

174
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
175
	eor	v14.16b,v14.16b,v4.16b
176
	eor	v13.16b,v14.16b,v13.16b
177
	// optimize sbox using AESE instruction
178
	tbl	v0.16b, {v13.16b}, v26.16b
179
	ushr	v24.16b, v0.16b, 4
180
	and	v0.16b, v0.16b, v31.16b
181
	tbl	v0.16b, {v28.16b}, v0.16b
182
	tbl	v24.16b, {v27.16b}, v24.16b
183
	eor	v0.16b, v0.16b, v24.16b
184
	eor	v1.16b, v1.16b, v1.16b
185
	aese	v0.16b,v1.16b
186
	ushr	v24.16b, v0.16b, 4
187
	and	v0.16b, v0.16b, v31.16b
188
	tbl	v0.16b, {v30.16b}, v0.16b
189
	tbl	v24.16b, {v29.16b}, v24.16b
190
	eor	v0.16b, v0.16b, v24.16b
191
	mov	v13.16b,v0.16b
192

193
	// linear transformation
194
	ushr	v0.4s,v13.4s,32-2
195
	ushr	v1.4s,v13.4s,32-10
196
	ushr	v2.4s,v13.4s,32-18
197
	ushr	v3.4s,v13.4s,32-24
198
	sli	v0.4s,v13.4s,2
199
	sli	v1.4s,v13.4s,10
200
	sli	v2.4s,v13.4s,18
201
	sli	v3.4s,v13.4s,24
202
	eor	v24.16b,v0.16b,v13.16b
203
	eor	v24.16b,v24.16b,v1.16b
204
	eor	v13.16b,v2.16b,v3.16b
205
	eor	v13.16b,v13.16b,v24.16b
206
	ldp	w7,w8,[x10],8
207
	eor	v5.16b,v5.16b,v13.16b
208

209
	dup	v12.4s,w7
210
	dup	v13.4s,w8
211

212
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
213
	eor	v14.16b,v4.16b,v5.16b
214
	eor	v12.16b,v7.16b,v12.16b
215
	eor	v12.16b,v14.16b,v12.16b
216
	// optimize sbox using AESE instruction
217
	tbl	v0.16b, {v12.16b}, v26.16b
218
	ushr	v24.16b, v0.16b, 4
219
	and	v0.16b, v0.16b, v31.16b
220
	tbl	v0.16b, {v28.16b}, v0.16b
221
	tbl	v24.16b, {v27.16b}, v24.16b
222
	eor	v0.16b, v0.16b, v24.16b
223
	eor	v1.16b, v1.16b, v1.16b
224
	aese	v0.16b,v1.16b
225
	ushr	v24.16b, v0.16b, 4
226
	and	v0.16b, v0.16b, v31.16b
227
	tbl	v0.16b, {v30.16b}, v0.16b
228
	tbl	v24.16b, {v29.16b}, v24.16b
229
	eor	v0.16b, v0.16b, v24.16b
230
	mov	v12.16b,v0.16b
231

232
	// linear transformation
233
	ushr	v0.4s,v12.4s,32-2
234
	ushr	v1.4s,v12.4s,32-10
235
	ushr	v2.4s,v12.4s,32-18
236
	ushr	v3.4s,v12.4s,32-24
237
	sli	v0.4s,v12.4s,2
238
	sli	v1.4s,v12.4s,10
239
	sli	v2.4s,v12.4s,18
240
	sli	v3.4s,v12.4s,24
241
	eor	v24.16b,v0.16b,v12.16b
242
	eor	v24.16b,v24.16b,v1.16b
243
	eor	v12.16b,v2.16b,v3.16b
244
	eor	v12.16b,v12.16b,v24.16b
245
	eor	v6.16b,v6.16b,v12.16b
246

247
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
248
	eor	v14.16b,v14.16b,v6.16b
249
	eor	v13.16b,v14.16b,v13.16b
250
	// optimize sbox using AESE instruction
251
	tbl	v0.16b, {v13.16b}, v26.16b
252
	ushr	v24.16b, v0.16b, 4
253
	and	v0.16b, v0.16b, v31.16b
254
	tbl	v0.16b, {v28.16b}, v0.16b
255
	tbl	v24.16b, {v27.16b}, v24.16b
256
	eor	v0.16b, v0.16b, v24.16b
257
	eor	v1.16b, v1.16b, v1.16b
258
	aese	v0.16b,v1.16b
259
	ushr	v24.16b, v0.16b, 4
260
	and	v0.16b, v0.16b, v31.16b
261
	tbl	v0.16b, {v30.16b}, v0.16b
262
	tbl	v24.16b, {v29.16b}, v24.16b
263
	eor	v0.16b, v0.16b, v24.16b
264
	mov	v13.16b,v0.16b
265

266
	// linear transformation
267
	ushr	v0.4s,v13.4s,32-2
268
	ushr	v1.4s,v13.4s,32-10
269
	ushr	v2.4s,v13.4s,32-18
270
	ushr	v3.4s,v13.4s,32-24
271
	sli	v0.4s,v13.4s,2
272
	sli	v1.4s,v13.4s,10
273
	sli	v2.4s,v13.4s,18
274
	sli	v3.4s,v13.4s,24
275
	eor	v24.16b,v0.16b,v13.16b
276
	eor	v24.16b,v24.16b,v1.16b
277
	eor	v13.16b,v2.16b,v3.16b
278
	eor	v13.16b,v13.16b,v24.16b
279
	eor	v7.16b,v7.16b,v13.16b
280
	subs	w11,w11,#1
281
	b.ne	10b
282
#ifndef __AARCH64EB__
283
	rev32	v3.16b,v4.16b
284
#else
285
	mov	v3.16b,v4.16b
286
#endif
287
#ifndef __AARCH64EB__
288
	rev32	v2.16b,v5.16b
289
#else
290
	mov	v2.16b,v5.16b
291
#endif
292
#ifndef __AARCH64EB__
293
	rev32	v1.16b,v6.16b
294
#else
295
	mov	v1.16b,v6.16b
296
#endif
297
#ifndef __AARCH64EB__
298
	rev32	v0.16b,v7.16b
299
#else
300
	mov	v0.16b,v7.16b
301
#endif
302
	ret
303
.size	_vpsm4_ex_enc_4blks,.-_vpsm4_ex_enc_4blks
304
.type	_vpsm4_ex_enc_8blks,%function
305
.align	4
306
_vpsm4_ex_enc_8blks:
307
	AARCH64_VALID_CALL_TARGET
308
	mov	x10,x3
309
	mov	w11,#8
310
10:
311
	ldp	w7,w8,[x10],8
312
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
313
	dup	v12.4s,w7
314
	eor	v14.16b,v6.16b,v7.16b
315
	eor	v15.16b,v10.16b,v11.16b
316
	eor	v0.16b,v5.16b,v12.16b
317
	eor	v1.16b,v9.16b,v12.16b
318
	eor	v12.16b,v14.16b,v0.16b
319
	eor	v13.16b,v15.16b,v1.16b
320
	// optimize sbox using AESE instruction
321
	tbl	v0.16b, {v12.16b}, v26.16b
322
	tbl	v1.16b, {v13.16b}, v26.16b
323
	ushr	v24.16b, v0.16b, 4
324
	and	v0.16b, v0.16b, v31.16b
325
	tbl	v0.16b, {v28.16b}, v0.16b
326
	tbl	v24.16b, {v27.16b}, v24.16b
327
	eor	v0.16b, v0.16b, v24.16b
328
	ushr	v24.16b, v1.16b, 4
329
	and	v1.16b, v1.16b, v31.16b
330
	tbl	v1.16b, {v28.16b}, v1.16b
331
	tbl	v24.16b, {v27.16b}, v24.16b
332
	eor	v1.16b, v1.16b, v24.16b
333
	eor	v25.16b, v25.16b, v25.16b
334
	aese	v0.16b,v25.16b
335
	aese	v1.16b,v25.16b
336
	ushr	v24.16b, v0.16b, 4
337
	and	v0.16b, v0.16b, v31.16b
338
	tbl	v0.16b, {v30.16b}, v0.16b
339
	tbl	v24.16b, {v29.16b}, v24.16b
340
	eor	v0.16b, v0.16b, v24.16b
341
	ushr	v24.16b, v1.16b, 4
342
	and	v1.16b, v1.16b, v31.16b
343
	tbl	v1.16b, {v30.16b}, v1.16b
344
	tbl	v24.16b, {v29.16b}, v24.16b
345
	eor	v1.16b, v1.16b, v24.16b
346
	mov	v12.16b,v0.16b
347
	mov	v13.16b,v1.16b
348

349
	// linear transformation
350
	ushr	v0.4s,v12.4s,32-2
351
	ushr	v25.4s,v13.4s,32-2
352
	ushr	v1.4s,v12.4s,32-10
353
	ushr	v2.4s,v12.4s,32-18
354
	ushr	v3.4s,v12.4s,32-24
355
	sli	v0.4s,v12.4s,2
356
	sli	v25.4s,v13.4s,2
357
	sli	v1.4s,v12.4s,10
358
	sli	v2.4s,v12.4s,18
359
	sli	v3.4s,v12.4s,24
360
	eor	v24.16b,v0.16b,v12.16b
361
	eor	v24.16b,v24.16b,v1.16b
362
	eor	v12.16b,v2.16b,v3.16b
363
	eor	v12.16b,v12.16b,v24.16b
364
	ushr	v1.4s,v13.4s,32-10
365
	ushr	v2.4s,v13.4s,32-18
366
	ushr	v3.4s,v13.4s,32-24
367
	sli	v1.4s,v13.4s,10
368
	sli	v2.4s,v13.4s,18
369
	sli	v3.4s,v13.4s,24
370
	eor	v24.16b,v25.16b,v13.16b
371
	eor	v24.16b,v24.16b,v1.16b
372
	eor	v13.16b,v2.16b,v3.16b
373
	eor	v13.16b,v13.16b,v24.16b
374
	eor	v4.16b,v4.16b,v12.16b
375
	eor	v8.16b,v8.16b,v13.16b
376

377
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
378
	dup	v13.4s,w8
379
	eor	v14.16b,v14.16b,v4.16b
380
	eor	v15.16b,v15.16b,v8.16b
381
	eor	v12.16b,v14.16b,v13.16b
382
	eor	v13.16b,v15.16b,v13.16b
383
	// optimize sbox using AESE instruction
384
	tbl	v0.16b, {v12.16b}, v26.16b
385
	tbl	v1.16b, {v13.16b}, v26.16b
386
	ushr	v24.16b, v0.16b, 4
387
	and	v0.16b, v0.16b, v31.16b
388
	tbl	v0.16b, {v28.16b}, v0.16b
389
	tbl	v24.16b, {v27.16b}, v24.16b
390
	eor	v0.16b, v0.16b, v24.16b
391
	ushr	v24.16b, v1.16b, 4
392
	and	v1.16b, v1.16b, v31.16b
393
	tbl	v1.16b, {v28.16b}, v1.16b
394
	tbl	v24.16b, {v27.16b}, v24.16b
395
	eor	v1.16b, v1.16b, v24.16b
396
	eor	v25.16b, v25.16b, v25.16b
397
	aese	v0.16b,v25.16b
398
	aese	v1.16b,v25.16b
399
	ushr	v24.16b, v0.16b, 4
400
	and	v0.16b, v0.16b, v31.16b
401
	tbl	v0.16b, {v30.16b}, v0.16b
402
	tbl	v24.16b, {v29.16b}, v24.16b
403
	eor	v0.16b, v0.16b, v24.16b
404
	ushr	v24.16b, v1.16b, 4
405
	and	v1.16b, v1.16b, v31.16b
406
	tbl	v1.16b, {v30.16b}, v1.16b
407
	tbl	v24.16b, {v29.16b}, v24.16b
408
	eor	v1.16b, v1.16b, v24.16b
409
	mov	v12.16b,v0.16b
410
	mov	v13.16b,v1.16b
411

412
	// linear transformation
413
	ushr	v0.4s,v12.4s,32-2
414
	ushr	v25.4s,v13.4s,32-2
415
	ushr	v1.4s,v12.4s,32-10
416
	ushr	v2.4s,v12.4s,32-18
417
	ushr	v3.4s,v12.4s,32-24
418
	sli	v0.4s,v12.4s,2
419
	sli	v25.4s,v13.4s,2
420
	sli	v1.4s,v12.4s,10
421
	sli	v2.4s,v12.4s,18
422
	sli	v3.4s,v12.4s,24
423
	eor	v24.16b,v0.16b,v12.16b
424
	eor	v24.16b,v24.16b,v1.16b
425
	eor	v12.16b,v2.16b,v3.16b
426
	eor	v12.16b,v12.16b,v24.16b
427
	ushr	v1.4s,v13.4s,32-10
428
	ushr	v2.4s,v13.4s,32-18
429
	ushr	v3.4s,v13.4s,32-24
430
	sli	v1.4s,v13.4s,10
431
	sli	v2.4s,v13.4s,18
432
	sli	v3.4s,v13.4s,24
433
	eor	v24.16b,v25.16b,v13.16b
434
	eor	v24.16b,v24.16b,v1.16b
435
	eor	v13.16b,v2.16b,v3.16b
436
	eor	v13.16b,v13.16b,v24.16b
437
	ldp	w7,w8,[x10],8
438
	eor	v5.16b,v5.16b,v12.16b
439
	eor	v9.16b,v9.16b,v13.16b
440

441
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
442
	dup	v12.4s,w7
443
	eor	v14.16b,v4.16b,v5.16b
444
	eor	v15.16b,v8.16b,v9.16b
445
	eor	v0.16b,v7.16b,v12.16b
446
	eor	v1.16b,v11.16b,v12.16b
447
	eor	v12.16b,v14.16b,v0.16b
448
	eor	v13.16b,v15.16b,v1.16b
449
	// optimize sbox using AESE instruction
450
	tbl	v0.16b, {v12.16b}, v26.16b
451
	tbl	v1.16b, {v13.16b}, v26.16b
452
	ushr	v24.16b, v0.16b, 4
453
	and	v0.16b, v0.16b, v31.16b
454
	tbl	v0.16b, {v28.16b}, v0.16b
455
	tbl	v24.16b, {v27.16b}, v24.16b
456
	eor	v0.16b, v0.16b, v24.16b
457
	ushr	v24.16b, v1.16b, 4
458
	and	v1.16b, v1.16b, v31.16b
459
	tbl	v1.16b, {v28.16b}, v1.16b
460
	tbl	v24.16b, {v27.16b}, v24.16b
461
	eor	v1.16b, v1.16b, v24.16b
462
	eor	v25.16b, v25.16b, v25.16b
463
	aese	v0.16b,v25.16b
464
	aese	v1.16b,v25.16b
465
	ushr	v24.16b, v0.16b, 4
466
	and	v0.16b, v0.16b, v31.16b
467
	tbl	v0.16b, {v30.16b}, v0.16b
468
	tbl	v24.16b, {v29.16b}, v24.16b
469
	eor	v0.16b, v0.16b, v24.16b
470
	ushr	v24.16b, v1.16b, 4
471
	and	v1.16b, v1.16b, v31.16b
472
	tbl	v1.16b, {v30.16b}, v1.16b
473
	tbl	v24.16b, {v29.16b}, v24.16b
474
	eor	v1.16b, v1.16b, v24.16b
475
	mov	v12.16b,v0.16b
476
	mov	v13.16b,v1.16b
477

478
	// linear transformation
479
	ushr	v0.4s,v12.4s,32-2
480
	ushr	v25.4s,v13.4s,32-2
481
	ushr	v1.4s,v12.4s,32-10
482
	ushr	v2.4s,v12.4s,32-18
483
	ushr	v3.4s,v12.4s,32-24
484
	sli	v0.4s,v12.4s,2
485
	sli	v25.4s,v13.4s,2
486
	sli	v1.4s,v12.4s,10
487
	sli	v2.4s,v12.4s,18
488
	sli	v3.4s,v12.4s,24
489
	eor	v24.16b,v0.16b,v12.16b
490
	eor	v24.16b,v24.16b,v1.16b
491
	eor	v12.16b,v2.16b,v3.16b
492
	eor	v12.16b,v12.16b,v24.16b
493
	ushr	v1.4s,v13.4s,32-10
494
	ushr	v2.4s,v13.4s,32-18
495
	ushr	v3.4s,v13.4s,32-24
496
	sli	v1.4s,v13.4s,10
497
	sli	v2.4s,v13.4s,18
498
	sli	v3.4s,v13.4s,24
499
	eor	v24.16b,v25.16b,v13.16b
500
	eor	v24.16b,v24.16b,v1.16b
501
	eor	v13.16b,v2.16b,v3.16b
502
	eor	v13.16b,v13.16b,v24.16b
503
	eor	v6.16b,v6.16b,v12.16b
504
	eor	v10.16b,v10.16b,v13.16b
505

506
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
507
	dup	v13.4s,w8
508
	eor	v14.16b,v14.16b,v6.16b
509
	eor	v15.16b,v15.16b,v10.16b
510
	eor	v12.16b,v14.16b,v13.16b
511
	eor	v13.16b,v15.16b,v13.16b
512
	// optimize sbox using AESE instruction
513
	tbl	v0.16b, {v12.16b}, v26.16b
514
	tbl	v1.16b, {v13.16b}, v26.16b
515
	ushr	v24.16b, v0.16b, 4
516
	and	v0.16b, v0.16b, v31.16b
517
	tbl	v0.16b, {v28.16b}, v0.16b
518
	tbl	v24.16b, {v27.16b}, v24.16b
519
	eor	v0.16b, v0.16b, v24.16b
520
	ushr	v24.16b, v1.16b, 4
521
	and	v1.16b, v1.16b, v31.16b
522
	tbl	v1.16b, {v28.16b}, v1.16b
523
	tbl	v24.16b, {v27.16b}, v24.16b
524
	eor	v1.16b, v1.16b, v24.16b
525
	eor	v25.16b, v25.16b, v25.16b
526
	aese	v0.16b,v25.16b
527
	aese	v1.16b,v25.16b
528
	ushr	v24.16b, v0.16b, 4
529
	and	v0.16b, v0.16b, v31.16b
530
	tbl	v0.16b, {v30.16b}, v0.16b
531
	tbl	v24.16b, {v29.16b}, v24.16b
532
	eor	v0.16b, v0.16b, v24.16b
533
	ushr	v24.16b, v1.16b, 4
534
	and	v1.16b, v1.16b, v31.16b
535
	tbl	v1.16b, {v30.16b}, v1.16b
536
	tbl	v24.16b, {v29.16b}, v24.16b
537
	eor	v1.16b, v1.16b, v24.16b
538
	mov	v12.16b,v0.16b
539
	mov	v13.16b,v1.16b
540

541
	// linear transformation
542
	ushr	v0.4s,v12.4s,32-2
543
	ushr	v25.4s,v13.4s,32-2
544
	ushr	v1.4s,v12.4s,32-10
545
	ushr	v2.4s,v12.4s,32-18
546
	ushr	v3.4s,v12.4s,32-24
547
	sli	v0.4s,v12.4s,2
548
	sli	v25.4s,v13.4s,2
549
	sli	v1.4s,v12.4s,10
550
	sli	v2.4s,v12.4s,18
551
	sli	v3.4s,v12.4s,24
552
	eor	v24.16b,v0.16b,v12.16b
553
	eor	v24.16b,v24.16b,v1.16b
554
	eor	v12.16b,v2.16b,v3.16b
555
	eor	v12.16b,v12.16b,v24.16b
556
	ushr	v1.4s,v13.4s,32-10
557
	ushr	v2.4s,v13.4s,32-18
558
	ushr	v3.4s,v13.4s,32-24
559
	sli	v1.4s,v13.4s,10
560
	sli	v2.4s,v13.4s,18
561
	sli	v3.4s,v13.4s,24
562
	eor	v24.16b,v25.16b,v13.16b
563
	eor	v24.16b,v24.16b,v1.16b
564
	eor	v13.16b,v2.16b,v3.16b
565
	eor	v13.16b,v13.16b,v24.16b
566
	eor	v7.16b,v7.16b,v12.16b
567
	eor	v11.16b,v11.16b,v13.16b
568
	subs	w11,w11,#1
569
	b.ne	10b
570
#ifndef __AARCH64EB__
571
	rev32	v3.16b,v4.16b
572
#else
573
	mov	v3.16b,v4.16b
574
#endif
575
#ifndef __AARCH64EB__
576
	rev32	v2.16b,v5.16b
577
#else
578
	mov	v2.16b,v5.16b
579
#endif
580
#ifndef __AARCH64EB__
581
	rev32	v1.16b,v6.16b
582
#else
583
	mov	v1.16b,v6.16b
584
#endif
585
#ifndef __AARCH64EB__
586
	rev32	v0.16b,v7.16b
587
#else
588
	mov	v0.16b,v7.16b
589
#endif
590
#ifndef __AARCH64EB__
591
	rev32	v7.16b,v8.16b
592
#else
593
	mov	v7.16b,v8.16b
594
#endif
595
#ifndef __AARCH64EB__
596
	rev32	v6.16b,v9.16b
597
#else
598
	mov	v6.16b,v9.16b
599
#endif
600
#ifndef __AARCH64EB__
601
	rev32	v5.16b,v10.16b
602
#else
603
	mov	v5.16b,v10.16b
604
#endif
605
#ifndef __AARCH64EB__
606
	rev32	v4.16b,v11.16b
607
#else
608
	mov	v4.16b,v11.16b
609
#endif
610
	ret
611
.size	_vpsm4_ex_enc_8blks,.-_vpsm4_ex_enc_8blks
612
.globl	vpsm4_ex_set_encrypt_key
613
.type	vpsm4_ex_set_encrypt_key,%function
614
.align	5
615
vpsm4_ex_set_encrypt_key:
616
	AARCH64_SIGN_LINK_REGISTER
617
	stp	x29,x30,[sp,#-16]!
618
	mov	w2,1
619
	bl	_vpsm4_ex_set_key
620
	ldp	x29,x30,[sp],#16
621
	AARCH64_VALIDATE_LINK_REGISTER
622
	ret
623
.size	vpsm4_ex_set_encrypt_key,.-vpsm4_ex_set_encrypt_key
624
.globl	vpsm4_ex_set_decrypt_key
625
.type	vpsm4_ex_set_decrypt_key,%function
626
.align	5
627
vpsm4_ex_set_decrypt_key:
628
	AARCH64_SIGN_LINK_REGISTER
629
	stp	x29,x30,[sp,#-16]!
630
	mov	w2,0
631
	bl	_vpsm4_ex_set_key
632
	ldp	x29,x30,[sp],#16
633
	AARCH64_VALIDATE_LINK_REGISTER
634
	ret
635
.size	vpsm4_ex_set_decrypt_key,.-vpsm4_ex_set_decrypt_key
636
.globl	vpsm4_ex_encrypt
637
.type	vpsm4_ex_encrypt,%function
638
.align	5
639
vpsm4_ex_encrypt:
640
	AARCH64_VALID_CALL_TARGET
641
	ld1	{v4.4s},[x0]
642
	adrp	x9, .Lsbox_magic
643
	ldr	q26, [x9, #:lo12:.Lsbox_magic]
644
	ldr	q27, [x9, #:lo12:.Lsbox_magic+16]
645
	ldr	q28, [x9, #:lo12:.Lsbox_magic+32]
646
	ldr	q29, [x9, #:lo12:.Lsbox_magic+48]
647
	ldr	q30, [x9, #:lo12:.Lsbox_magic+64]
648
	ldr	q31, [x9, #:lo12:.Lsbox_magic+80]
649
#ifndef __AARCH64EB__
650
	rev32	v4.16b,v4.16b
651
#endif
652
	mov	x3,x2
653
	mov	x10,x3
654
	mov	w11,#8
655
	mov	w12,v4.s[0]
656
	mov	w13,v4.s[1]
657
	mov	w14,v4.s[2]
658
	mov	w15,v4.s[3]
659
10:
660
	ldp	w7,w8,[x10],8
661
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
662
	eor	w6,w14,w15
663
	eor	w9,w7,w13
664
	eor	w6,w6,w9
665
	mov	v3.s[0],w6
666
	// optimize sbox using AESE instruction
667
	tbl	v0.16b, {v3.16b}, v26.16b
668
	ushr	v2.16b, v0.16b, 4
669
	and	v0.16b, v0.16b, v31.16b
670
	tbl	v0.16b, {v28.16b}, v0.16b
671
	tbl	v2.16b, {v27.16b}, v2.16b
672
	eor	v0.16b, v0.16b, v2.16b
673
	eor	v1.16b, v1.16b, v1.16b
674
	aese	v0.16b,v1.16b
675
	ushr	v2.16b, v0.16b, 4
676
	and	v0.16b, v0.16b, v31.16b
677
	tbl	v0.16b, {v30.16b}, v0.16b
678
	tbl	v2.16b, {v29.16b}, v2.16b
679
	eor	v0.16b, v0.16b, v2.16b
680

681
	mov	w7,v0.s[0]
682
	eor	w6,w7,w7,ror #32-2
683
	eor	w6,w6,w7,ror #32-10
684
	eor	w6,w6,w7,ror #32-18
685
	eor	w6,w6,w7,ror #32-24
686
	eor	w12,w12,w6
687
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
688
	eor	w6,w14,w15
689
	eor	w9,w12,w8
690
	eor	w6,w6,w9
691
	mov	v3.s[0],w6
692
	// optimize sbox using AESE instruction
693
	tbl	v0.16b, {v3.16b}, v26.16b
694
	ushr	v2.16b, v0.16b, 4
695
	and	v0.16b, v0.16b, v31.16b
696
	tbl	v0.16b, {v28.16b}, v0.16b
697
	tbl	v2.16b, {v27.16b}, v2.16b
698
	eor	v0.16b, v0.16b, v2.16b
699
	eor	v1.16b, v1.16b, v1.16b
700
	aese	v0.16b,v1.16b
701
	ushr	v2.16b, v0.16b, 4
702
	and	v0.16b, v0.16b, v31.16b
703
	tbl	v0.16b, {v30.16b}, v0.16b
704
	tbl	v2.16b, {v29.16b}, v2.16b
705
	eor	v0.16b, v0.16b, v2.16b
706

707
	mov	w7,v0.s[0]
708
	eor	w6,w7,w7,ror #32-2
709
	eor	w6,w6,w7,ror #32-10
710
	eor	w6,w6,w7,ror #32-18
711
	eor	w6,w6,w7,ror #32-24
712
	ldp	w7,w8,[x10],8
713
	eor	w13,w13,w6
714
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
715
	eor	w6,w12,w13
716
	eor	w9,w7,w15
717
	eor	w6,w6,w9
718
	mov	v3.s[0],w6
719
	// optimize sbox using AESE instruction
720
	tbl	v0.16b, {v3.16b}, v26.16b
721
	ushr	v2.16b, v0.16b, 4
722
	and	v0.16b, v0.16b, v31.16b
723
	tbl	v0.16b, {v28.16b}, v0.16b
724
	tbl	v2.16b, {v27.16b}, v2.16b
725
	eor	v0.16b, v0.16b, v2.16b
726
	eor	v1.16b, v1.16b, v1.16b
727
	aese	v0.16b,v1.16b
728
	ushr	v2.16b, v0.16b, 4
729
	and	v0.16b, v0.16b, v31.16b
730
	tbl	v0.16b, {v30.16b}, v0.16b
731
	tbl	v2.16b, {v29.16b}, v2.16b
732
	eor	v0.16b, v0.16b, v2.16b
733

734
	mov	w7,v0.s[0]
735
	eor	w6,w7,w7,ror #32-2
736
	eor	w6,w6,w7,ror #32-10
737
	eor	w6,w6,w7,ror #32-18
738
	eor	w6,w6,w7,ror #32-24
739
	eor	w14,w14,w6
740
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
741
	eor	w6,w12,w13
742
	eor	w9,w14,w8
743
	eor	w6,w6,w9
744
	mov	v3.s[0],w6
745
	// optimize sbox using AESE instruction
746
	tbl	v0.16b, {v3.16b}, v26.16b
747
	ushr	v2.16b, v0.16b, 4
748
	and	v0.16b, v0.16b, v31.16b
749
	tbl	v0.16b, {v28.16b}, v0.16b
750
	tbl	v2.16b, {v27.16b}, v2.16b
751
	eor	v0.16b, v0.16b, v2.16b
752
	eor	v1.16b, v1.16b, v1.16b
753
	aese	v0.16b,v1.16b
754
	ushr	v2.16b, v0.16b, 4
755
	and	v0.16b, v0.16b, v31.16b
756
	tbl	v0.16b, {v30.16b}, v0.16b
757
	tbl	v2.16b, {v29.16b}, v2.16b
758
	eor	v0.16b, v0.16b, v2.16b
759

760
	mov	w7,v0.s[0]
761
	eor	w6,w7,w7,ror #32-2
762
	eor	w6,w6,w7,ror #32-10
763
	eor	w6,w6,w7,ror #32-18
764
	eor	w6,w6,w7,ror #32-24
765
	eor	w15,w15,w6
766
	subs	w11,w11,#1
767
	b.ne	10b
768
	mov	v4.s[0],w15
769
	mov	v4.s[1],w14
770
	mov	v4.s[2],w13
771
	mov	v4.s[3],w12
772
#ifndef __AARCH64EB__
773
	rev32	v4.16b,v4.16b
774
#endif
775
	st1	{v4.4s},[x1]
776
	ret
777
.size	vpsm4_ex_encrypt,.-vpsm4_ex_encrypt
778
.globl	vpsm4_ex_decrypt
779
.type	vpsm4_ex_decrypt,%function
780
.align	5
781
vpsm4_ex_decrypt:
782
	AARCH64_VALID_CALL_TARGET
783
	ld1	{v4.4s},[x0]
784
	adrp	x9, .Lsbox_magic
785
	ldr	q26, [x9, #:lo12:.Lsbox_magic]
786
	ldr	q27, [x9, #:lo12:.Lsbox_magic+16]
787
	ldr	q28, [x9, #:lo12:.Lsbox_magic+32]
788
	ldr	q29, [x9, #:lo12:.Lsbox_magic+48]
789
	ldr	q30, [x9, #:lo12:.Lsbox_magic+64]
790
	ldr	q31, [x9, #:lo12:.Lsbox_magic+80]
791
#ifndef __AARCH64EB__
792
	rev32	v4.16b,v4.16b
793
#endif
794
	mov	x3,x2
795
	mov	x10,x3
796
	mov	w11,#8
797
	mov	w12,v4.s[0]
798
	mov	w13,v4.s[1]
799
	mov	w14,v4.s[2]
800
	mov	w15,v4.s[3]
801
10:
802
	ldp	w7,w8,[x10],8
803
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
804
	eor	w6,w14,w15
805
	eor	w9,w7,w13
806
	eor	w6,w6,w9
807
	mov	v3.s[0],w6
808
	// optimize sbox using AESE instruction
809
	tbl	v0.16b, {v3.16b}, v26.16b
810
	ushr	v2.16b, v0.16b, 4
811
	and	v0.16b, v0.16b, v31.16b
812
	tbl	v0.16b, {v28.16b}, v0.16b
813
	tbl	v2.16b, {v27.16b}, v2.16b
814
	eor	v0.16b, v0.16b, v2.16b
815
	eor	v1.16b, v1.16b, v1.16b
816
	aese	v0.16b,v1.16b
817
	ushr	v2.16b, v0.16b, 4
818
	and	v0.16b, v0.16b, v31.16b
819
	tbl	v0.16b, {v30.16b}, v0.16b
820
	tbl	v2.16b, {v29.16b}, v2.16b
821
	eor	v0.16b, v0.16b, v2.16b
822

823
	mov	w7,v0.s[0]
824
	eor	w6,w7,w7,ror #32-2
825
	eor	w6,w6,w7,ror #32-10
826
	eor	w6,w6,w7,ror #32-18
827
	eor	w6,w6,w7,ror #32-24
828
	eor	w12,w12,w6
829
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
830
	eor	w6,w14,w15
831
	eor	w9,w12,w8
832
	eor	w6,w6,w9
833
	mov	v3.s[0],w6
834
	// optimize sbox using AESE instruction
835
	tbl	v0.16b, {v3.16b}, v26.16b
836
	ushr	v2.16b, v0.16b, 4
837
	and	v0.16b, v0.16b, v31.16b
838
	tbl	v0.16b, {v28.16b}, v0.16b
839
	tbl	v2.16b, {v27.16b}, v2.16b
840
	eor	v0.16b, v0.16b, v2.16b
841
	eor	v1.16b, v1.16b, v1.16b
842
	aese	v0.16b,v1.16b
843
	ushr	v2.16b, v0.16b, 4
844
	and	v0.16b, v0.16b, v31.16b
845
	tbl	v0.16b, {v30.16b}, v0.16b
846
	tbl	v2.16b, {v29.16b}, v2.16b
847
	eor	v0.16b, v0.16b, v2.16b
848

849
	mov	w7,v0.s[0]
850
	eor	w6,w7,w7,ror #32-2
851
	eor	w6,w6,w7,ror #32-10
852
	eor	w6,w6,w7,ror #32-18
853
	eor	w6,w6,w7,ror #32-24
854
	ldp	w7,w8,[x10],8
855
	eor	w13,w13,w6
856
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
857
	eor	w6,w12,w13
858
	eor	w9,w7,w15
859
	eor	w6,w6,w9
860
	mov	v3.s[0],w6
861
	// optimize sbox using AESE instruction
862
	tbl	v0.16b, {v3.16b}, v26.16b
863
	ushr	v2.16b, v0.16b, 4
864
	and	v0.16b, v0.16b, v31.16b
865
	tbl	v0.16b, {v28.16b}, v0.16b
866
	tbl	v2.16b, {v27.16b}, v2.16b
867
	eor	v0.16b, v0.16b, v2.16b
868
	eor	v1.16b, v1.16b, v1.16b
869
	aese	v0.16b,v1.16b
870
	ushr	v2.16b, v0.16b, 4
871
	and	v0.16b, v0.16b, v31.16b
872
	tbl	v0.16b, {v30.16b}, v0.16b
873
	tbl	v2.16b, {v29.16b}, v2.16b
874
	eor	v0.16b, v0.16b, v2.16b
875

876
	mov	w7,v0.s[0]
877
	eor	w6,w7,w7,ror #32-2
878
	eor	w6,w6,w7,ror #32-10
879
	eor	w6,w6,w7,ror #32-18
880
	eor	w6,w6,w7,ror #32-24
881
	eor	w14,w14,w6
882
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
883
	eor	w6,w12,w13
884
	eor	w9,w14,w8
885
	eor	w6,w6,w9
886
	mov	v3.s[0],w6
887
	// optimize sbox using AESE instruction
888
	tbl	v0.16b, {v3.16b}, v26.16b
889
	ushr	v2.16b, v0.16b, 4
890
	and	v0.16b, v0.16b, v31.16b
891
	tbl	v0.16b, {v28.16b}, v0.16b
892
	tbl	v2.16b, {v27.16b}, v2.16b
893
	eor	v0.16b, v0.16b, v2.16b
894
	eor	v1.16b, v1.16b, v1.16b
895
	aese	v0.16b,v1.16b
896
	ushr	v2.16b, v0.16b, 4
897
	and	v0.16b, v0.16b, v31.16b
898
	tbl	v0.16b, {v30.16b}, v0.16b
899
	tbl	v2.16b, {v29.16b}, v2.16b
900
	eor	v0.16b, v0.16b, v2.16b
901

902
	mov	w7,v0.s[0]
903
	eor	w6,w7,w7,ror #32-2
904
	eor	w6,w6,w7,ror #32-10
905
	eor	w6,w6,w7,ror #32-18
906
	eor	w6,w6,w7,ror #32-24
907
	eor	w15,w15,w6
908
	subs	w11,w11,#1
909
	b.ne	10b
910
	mov	v4.s[0],w15
911
	mov	v4.s[1],w14
912
	mov	v4.s[2],w13
913
	mov	v4.s[3],w12
914
#ifndef __AARCH64EB__
915
	rev32	v4.16b,v4.16b
916
#endif
917
	st1	{v4.4s},[x1]
918
	ret
919
.size	vpsm4_ex_decrypt,.-vpsm4_ex_decrypt
920
.globl	vpsm4_ex_ecb_encrypt
921
.type	vpsm4_ex_ecb_encrypt,%function
922
.align	5
923
vpsm4_ex_ecb_encrypt:
924
	AARCH64_SIGN_LINK_REGISTER
925
	// convert length into blocks
926
	lsr	x2,x2,4
927
	stp	d8,d9,[sp,#-80]!
928
	stp	d10,d11,[sp,#16]
929
	stp	d12,d13,[sp,#32]
930
	stp	d14,d15,[sp,#48]
931
	stp	x29,x30,[sp,#64]
932
	adrp	x9, .Lsbox_magic
933
	ldr	q26, [x9, #:lo12:.Lsbox_magic]
934
	ldr	q27, [x9, #:lo12:.Lsbox_magic+16]
935
	ldr	q28, [x9, #:lo12:.Lsbox_magic+32]
936
	ldr	q29, [x9, #:lo12:.Lsbox_magic+48]
937
	ldr	q30, [x9, #:lo12:.Lsbox_magic+64]
938
	ldr	q31, [x9, #:lo12:.Lsbox_magic+80]
939
.Lecb_8_blocks_process:
940
	cmp	w2,#8
941
	b.lt	.Lecb_4_blocks_process
942
	ld4	{v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64
943
	ld4	{v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64
944
#ifndef __AARCH64EB__
945
	rev32	v4.16b,v4.16b
946
#endif
947
#ifndef __AARCH64EB__
948
	rev32	v5.16b,v5.16b
949
#endif
950
#ifndef __AARCH64EB__
951
	rev32	v6.16b,v6.16b
952
#endif
953
#ifndef __AARCH64EB__
954
	rev32	v7.16b,v7.16b
955
#endif
956
#ifndef __AARCH64EB__
957
	rev32	v8.16b,v8.16b
958
#endif
959
#ifndef __AARCH64EB__
960
	rev32	v9.16b,v9.16b
961
#endif
962
#ifndef __AARCH64EB__
963
	rev32	v10.16b,v10.16b
964
#endif
965
#ifndef __AARCH64EB__
966
	rev32	v11.16b,v11.16b
967
#endif
968
	bl	_vpsm4_ex_enc_8blks
969
	st4	{v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64
970
	st4	{v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64
971
	subs	w2,w2,#8
972
	b.gt	.Lecb_8_blocks_process
973
	b	100f
974
.Lecb_4_blocks_process:
975
	cmp	w2,#4
976
	b.lt	1f
977
	ld4	{v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64
978
#ifndef __AARCH64EB__
979
	rev32	v4.16b,v4.16b
980
#endif
981
#ifndef __AARCH64EB__
982
	rev32	v5.16b,v5.16b
983
#endif
984
#ifndef __AARCH64EB__
985
	rev32	v6.16b,v6.16b
986
#endif
987
#ifndef __AARCH64EB__
988
	rev32	v7.16b,v7.16b
989
#endif
990
	bl	_vpsm4_ex_enc_4blks
991
	st4	{v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64
992
	sub	w2,w2,#4
993
1:
994
	// process last block
995
	cmp	w2,#1
996
	b.lt	100f
997
	b.gt	1f
998
	ld1	{v4.4s},[x0]
999
#ifndef __AARCH64EB__
1000
	rev32	v4.16b,v4.16b
1001
#endif
1002
	mov	x10,x3
1003
	mov	w11,#8
1004
	mov	w12,v4.s[0]
1005
	mov	w13,v4.s[1]
1006
	mov	w14,v4.s[2]
1007
	mov	w15,v4.s[3]
1008
10:
1009
	ldp	w7,w8,[x10],8
1010
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
1011
	eor	w6,w14,w15
1012
	eor	w9,w7,w13
1013
	eor	w6,w6,w9
1014
	mov	v3.s[0],w6
1015
	// optimize sbox using AESE instruction
1016
	tbl	v0.16b, {v3.16b}, v26.16b
1017
	ushr	v2.16b, v0.16b, 4
1018
	and	v0.16b, v0.16b, v31.16b
1019
	tbl	v0.16b, {v28.16b}, v0.16b
1020
	tbl	v2.16b, {v27.16b}, v2.16b
1021
	eor	v0.16b, v0.16b, v2.16b
1022
	eor	v1.16b, v1.16b, v1.16b
1023
	aese	v0.16b,v1.16b
1024
	ushr	v2.16b, v0.16b, 4
1025
	and	v0.16b, v0.16b, v31.16b
1026
	tbl	v0.16b, {v30.16b}, v0.16b
1027
	tbl	v2.16b, {v29.16b}, v2.16b
1028
	eor	v0.16b, v0.16b, v2.16b
1029

1030
	mov	w7,v0.s[0]
1031
	eor	w6,w7,w7,ror #32-2
1032
	eor	w6,w6,w7,ror #32-10
1033
	eor	w6,w6,w7,ror #32-18
1034
	eor	w6,w6,w7,ror #32-24
1035
	eor	w12,w12,w6
1036
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
1037
	eor	w6,w14,w15
1038
	eor	w9,w12,w8
1039
	eor	w6,w6,w9
1040
	mov	v3.s[0],w6
1041
	// optimize sbox using AESE instruction
1042
	tbl	v0.16b, {v3.16b}, v26.16b
1043
	ushr	v2.16b, v0.16b, 4
1044
	and	v0.16b, v0.16b, v31.16b
1045
	tbl	v0.16b, {v28.16b}, v0.16b
1046
	tbl	v2.16b, {v27.16b}, v2.16b
1047
	eor	v0.16b, v0.16b, v2.16b
1048
	eor	v1.16b, v1.16b, v1.16b
1049
	aese	v0.16b,v1.16b
1050
	ushr	v2.16b, v0.16b, 4
1051
	and	v0.16b, v0.16b, v31.16b
1052
	tbl	v0.16b, {v30.16b}, v0.16b
1053
	tbl	v2.16b, {v29.16b}, v2.16b
1054
	eor	v0.16b, v0.16b, v2.16b
1055

1056
	mov	w7,v0.s[0]
1057
	eor	w6,w7,w7,ror #32-2
1058
	eor	w6,w6,w7,ror #32-10
1059
	eor	w6,w6,w7,ror #32-18
1060
	eor	w6,w6,w7,ror #32-24
1061
	ldp	w7,w8,[x10],8
1062
	eor	w13,w13,w6
1063
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
1064
	eor	w6,w12,w13
1065
	eor	w9,w7,w15
1066
	eor	w6,w6,w9
1067
	mov	v3.s[0],w6
1068
	// optimize sbox using AESE instruction
1069
	tbl	v0.16b, {v3.16b}, v26.16b
1070
	ushr	v2.16b, v0.16b, 4
1071
	and	v0.16b, v0.16b, v31.16b
1072
	tbl	v0.16b, {v28.16b}, v0.16b
1073
	tbl	v2.16b, {v27.16b}, v2.16b
1074
	eor	v0.16b, v0.16b, v2.16b
1075
	eor	v1.16b, v1.16b, v1.16b
1076
	aese	v0.16b,v1.16b
1077
	ushr	v2.16b, v0.16b, 4
1078
	and	v0.16b, v0.16b, v31.16b
1079
	tbl	v0.16b, {v30.16b}, v0.16b
1080
	tbl	v2.16b, {v29.16b}, v2.16b
1081
	eor	v0.16b, v0.16b, v2.16b
1082

1083
	mov	w7,v0.s[0]
1084
	eor	w6,w7,w7,ror #32-2
1085
	eor	w6,w6,w7,ror #32-10
1086
	eor	w6,w6,w7,ror #32-18
1087
	eor	w6,w6,w7,ror #32-24
1088
	eor	w14,w14,w6
1089
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
1090
	eor	w6,w12,w13
1091
	eor	w9,w14,w8
1092
	eor	w6,w6,w9
1093
	mov	v3.s[0],w6
1094
	// optimize sbox using AESE instruction
1095
	tbl	v0.16b, {v3.16b}, v26.16b
1096
	ushr	v2.16b, v0.16b, 4
1097
	and	v0.16b, v0.16b, v31.16b
1098
	tbl	v0.16b, {v28.16b}, v0.16b
1099
	tbl	v2.16b, {v27.16b}, v2.16b
1100
	eor	v0.16b, v0.16b, v2.16b
1101
	eor	v1.16b, v1.16b, v1.16b
1102
	aese	v0.16b,v1.16b
1103
	ushr	v2.16b, v0.16b, 4
1104
	and	v0.16b, v0.16b, v31.16b
1105
	tbl	v0.16b, {v30.16b}, v0.16b
1106
	tbl	v2.16b, {v29.16b}, v2.16b
1107
	eor	v0.16b, v0.16b, v2.16b
1108

1109
	mov	w7,v0.s[0]
1110
	eor	w6,w7,w7,ror #32-2
1111
	eor	w6,w6,w7,ror #32-10
1112
	eor	w6,w6,w7,ror #32-18
1113
	eor	w6,w6,w7,ror #32-24
1114
	eor	w15,w15,w6
1115
	subs	w11,w11,#1
1116
	b.ne	10b
1117
	mov	v4.s[0],w15
1118
	mov	v4.s[1],w14
1119
	mov	v4.s[2],w13
1120
	mov	v4.s[3],w12
1121
#ifndef __AARCH64EB__
1122
	rev32	v4.16b,v4.16b
1123
#endif
1124
	st1	{v4.4s},[x1]
1125
	b	100f
1126
1:	//	process last 2 blocks
1127
	ld4	{v4.s,v5.s,v6.s,v7.s}[0],[x0],#16
1128
	ld4	{v4.s,v5.s,v6.s,v7.s}[1],[x0],#16
1129
	cmp	w2,#2
1130
	b.gt	1f
1131
#ifndef __AARCH64EB__
1132
	rev32	v4.16b,v4.16b
1133
#endif
1134
#ifndef __AARCH64EB__
1135
	rev32	v5.16b,v5.16b
1136
#endif
1137
#ifndef __AARCH64EB__
1138
	rev32	v6.16b,v6.16b
1139
#endif
1140
#ifndef __AARCH64EB__
1141
	rev32	v7.16b,v7.16b
1142
#endif
1143
	bl	_vpsm4_ex_enc_4blks
1144
	st4	{v0.s,v1.s,v2.s,v3.s}[0],[x1],#16
1145
	st4	{v0.s,v1.s,v2.s,v3.s}[1],[x1]
1146
	b	100f
1147
1:	//	process last 3 blocks
1148
	ld4	{v4.s,v5.s,v6.s,v7.s}[2],[x0],#16
1149
#ifndef __AARCH64EB__
1150
	rev32	v4.16b,v4.16b
1151
#endif
1152
#ifndef __AARCH64EB__
1153
	rev32	v5.16b,v5.16b
1154
#endif
1155
#ifndef __AARCH64EB__
1156
	rev32	v6.16b,v6.16b
1157
#endif
1158
#ifndef __AARCH64EB__
1159
	rev32	v7.16b,v7.16b
1160
#endif
1161
	bl	_vpsm4_ex_enc_4blks
1162
	st4	{v0.s,v1.s,v2.s,v3.s}[0],[x1],#16
1163
	st4	{v0.s,v1.s,v2.s,v3.s}[1],[x1],#16
1164
	st4	{v0.s,v1.s,v2.s,v3.s}[2],[x1]
1165
100:
1166
	ldp	d10,d11,[sp,#16]
1167
	ldp	d12,d13,[sp,#32]
1168
	ldp	d14,d15,[sp,#48]
1169
	ldp	x29,x30,[sp,#64]
1170
	ldp	d8,d9,[sp],#80
1171
	AARCH64_VALIDATE_LINK_REGISTER
1172
	ret
1173
.size	vpsm4_ex_ecb_encrypt,.-vpsm4_ex_ecb_encrypt
1174
.globl	vpsm4_ex_cbc_encrypt
1175
.type	vpsm4_ex_cbc_encrypt,%function
1176
.align	5
1177
vpsm4_ex_cbc_encrypt:
1178
	AARCH64_VALID_CALL_TARGET
1179
	lsr	x2,x2,4
1180
	adrp	x9, .Lsbox_magic
1181
	ldr	q26, [x9, #:lo12:.Lsbox_magic]
1182
	ldr	q27, [x9, #:lo12:.Lsbox_magic+16]
1183
	ldr	q28, [x9, #:lo12:.Lsbox_magic+32]
1184
	ldr	q29, [x9, #:lo12:.Lsbox_magic+48]
1185
	ldr	q30, [x9, #:lo12:.Lsbox_magic+64]
1186
	ldr	q31, [x9, #:lo12:.Lsbox_magic+80]
1187
	cbz	w5,.Ldec
1188
	ld1	{v3.4s},[x4]
1189
.Lcbc_4_blocks_enc:
1190
	cmp	w2,#4
1191
	b.lt	1f
1192
	ld1	{v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64
1193
	eor	v4.16b,v4.16b,v3.16b
1194
#ifndef __AARCH64EB__
1195
	rev32	v5.16b,v5.16b
1196
#endif
1197
#ifndef __AARCH64EB__
1198
	rev32	v4.16b,v4.16b
1199
#endif
1200
#ifndef __AARCH64EB__
1201
	rev32	v6.16b,v6.16b
1202
#endif
1203
#ifndef __AARCH64EB__
1204
	rev32	v7.16b,v7.16b
1205
#endif
1206
	mov	x10,x3
1207
	mov	w11,#8
1208
	mov	w12,v4.s[0]
1209
	mov	w13,v4.s[1]
1210
	mov	w14,v4.s[2]
1211
	mov	w15,v4.s[3]
1212
10:
1213
	ldp	w7,w8,[x10],8
1214
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
1215
	eor	w6,w14,w15
1216
	eor	w9,w7,w13
1217
	eor	w6,w6,w9
1218
	mov	v3.s[0],w6
1219
	// optimize sbox using AESE instruction
1220
	tbl	v0.16b, {v3.16b}, v26.16b
1221
	ushr	v2.16b, v0.16b, 4
1222
	and	v0.16b, v0.16b, v31.16b
1223
	tbl	v0.16b, {v28.16b}, v0.16b
1224
	tbl	v2.16b, {v27.16b}, v2.16b
1225
	eor	v0.16b, v0.16b, v2.16b
1226
	eor	v1.16b, v1.16b, v1.16b
1227
	aese	v0.16b,v1.16b
1228
	ushr	v2.16b, v0.16b, 4
1229
	and	v0.16b, v0.16b, v31.16b
1230
	tbl	v0.16b, {v30.16b}, v0.16b
1231
	tbl	v2.16b, {v29.16b}, v2.16b
1232
	eor	v0.16b, v0.16b, v2.16b
1233

1234
	mov	w7,v0.s[0]
1235
	eor	w6,w7,w7,ror #32-2
1236
	eor	w6,w6,w7,ror #32-10
1237
	eor	w6,w6,w7,ror #32-18
1238
	eor	w6,w6,w7,ror #32-24
1239
	eor	w12,w12,w6
1240
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
1241
	eor	w6,w14,w15
1242
	eor	w9,w12,w8
1243
	eor	w6,w6,w9
1244
	mov	v3.s[0],w6
1245
	// optimize sbox using AESE instruction
1246
	tbl	v0.16b, {v3.16b}, v26.16b
1247
	ushr	v2.16b, v0.16b, 4
1248
	and	v0.16b, v0.16b, v31.16b
1249
	tbl	v0.16b, {v28.16b}, v0.16b
1250
	tbl	v2.16b, {v27.16b}, v2.16b
1251
	eor	v0.16b, v0.16b, v2.16b
1252
	eor	v1.16b, v1.16b, v1.16b
1253
	aese	v0.16b,v1.16b
1254
	ushr	v2.16b, v0.16b, 4
1255
	and	v0.16b, v0.16b, v31.16b
1256
	tbl	v0.16b, {v30.16b}, v0.16b
1257
	tbl	v2.16b, {v29.16b}, v2.16b
1258
	eor	v0.16b, v0.16b, v2.16b
1259

1260
	mov	w7,v0.s[0]
1261
	eor	w6,w7,w7,ror #32-2
1262
	eor	w6,w6,w7,ror #32-10
1263
	eor	w6,w6,w7,ror #32-18
1264
	eor	w6,w6,w7,ror #32-24
1265
	ldp	w7,w8,[x10],8
1266
	eor	w13,w13,w6
1267
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
1268
	eor	w6,w12,w13
1269
	eor	w9,w7,w15
1270
	eor	w6,w6,w9
1271
	mov	v3.s[0],w6
1272
	// optimize sbox using AESE instruction
1273
	tbl	v0.16b, {v3.16b}, v26.16b
1274
	ushr	v2.16b, v0.16b, 4
1275
	and	v0.16b, v0.16b, v31.16b
1276
	tbl	v0.16b, {v28.16b}, v0.16b
1277
	tbl	v2.16b, {v27.16b}, v2.16b
1278
	eor	v0.16b, v0.16b, v2.16b
1279
	eor	v1.16b, v1.16b, v1.16b
1280
	aese	v0.16b,v1.16b
1281
	ushr	v2.16b, v0.16b, 4
1282
	and	v0.16b, v0.16b, v31.16b
1283
	tbl	v0.16b, {v30.16b}, v0.16b
1284
	tbl	v2.16b, {v29.16b}, v2.16b
1285
	eor	v0.16b, v0.16b, v2.16b
1286

1287
	mov	w7,v0.s[0]
1288
	eor	w6,w7,w7,ror #32-2
1289
	eor	w6,w6,w7,ror #32-10
1290
	eor	w6,w6,w7,ror #32-18
1291
	eor	w6,w6,w7,ror #32-24
1292
	eor	w14,w14,w6
1293
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
1294
	eor	w6,w12,w13
1295
	eor	w9,w14,w8
1296
	eor	w6,w6,w9
1297
	mov	v3.s[0],w6
1298
	// optimize sbox using AESE instruction
1299
	tbl	v0.16b, {v3.16b}, v26.16b
1300
	ushr	v2.16b, v0.16b, 4
1301
	and	v0.16b, v0.16b, v31.16b
1302
	tbl	v0.16b, {v28.16b}, v0.16b
1303
	tbl	v2.16b, {v27.16b}, v2.16b
1304
	eor	v0.16b, v0.16b, v2.16b
1305
	eor	v1.16b, v1.16b, v1.16b
1306
	aese	v0.16b,v1.16b
1307
	ushr	v2.16b, v0.16b, 4
1308
	and	v0.16b, v0.16b, v31.16b
1309
	tbl	v0.16b, {v30.16b}, v0.16b
1310
	tbl	v2.16b, {v29.16b}, v2.16b
1311
	eor	v0.16b, v0.16b, v2.16b
1312

1313
	mov	w7,v0.s[0]
1314
	eor	w6,w7,w7,ror #32-2
1315
	eor	w6,w6,w7,ror #32-10
1316
	eor	w6,w6,w7,ror #32-18
1317
	eor	w6,w6,w7,ror #32-24
1318
	eor	w15,w15,w6
1319
	subs	w11,w11,#1
1320
	b.ne	10b
1321
	mov	v4.s[0],w15
1322
	mov	v4.s[1],w14
1323
	mov	v4.s[2],w13
1324
	mov	v4.s[3],w12
1325
	eor	v5.16b,v5.16b,v4.16b
1326
	mov	x10,x3
1327
	mov	w11,#8
1328
	mov	w12,v5.s[0]
1329
	mov	w13,v5.s[1]
1330
	mov	w14,v5.s[2]
1331
	mov	w15,v5.s[3]
1332
10:
1333
	ldp	w7,w8,[x10],8
1334
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
1335
	eor	w6,w14,w15
1336
	eor	w9,w7,w13
1337
	eor	w6,w6,w9
1338
	mov	v3.s[0],w6
1339
	// optimize sbox using AESE instruction
1340
	tbl	v0.16b, {v3.16b}, v26.16b
1341
	ushr	v2.16b, v0.16b, 4
1342
	and	v0.16b, v0.16b, v31.16b
1343
	tbl	v0.16b, {v28.16b}, v0.16b
1344
	tbl	v2.16b, {v27.16b}, v2.16b
1345
	eor	v0.16b, v0.16b, v2.16b
1346
	eor	v1.16b, v1.16b, v1.16b
1347
	aese	v0.16b,v1.16b
1348
	ushr	v2.16b, v0.16b, 4
1349
	and	v0.16b, v0.16b, v31.16b
1350
	tbl	v0.16b, {v30.16b}, v0.16b
1351
	tbl	v2.16b, {v29.16b}, v2.16b
1352
	eor	v0.16b, v0.16b, v2.16b
1353

1354
	mov	w7,v0.s[0]
1355
	eor	w6,w7,w7,ror #32-2
1356
	eor	w6,w6,w7,ror #32-10
1357
	eor	w6,w6,w7,ror #32-18
1358
	eor	w6,w6,w7,ror #32-24
1359
	eor	w12,w12,w6
1360
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
1361
	eor	w6,w14,w15
1362
	eor	w9,w12,w8
1363
	eor	w6,w6,w9
1364
	mov	v3.s[0],w6
1365
	// optimize sbox using AESE instruction
1366
	tbl	v0.16b, {v3.16b}, v26.16b
1367
	ushr	v2.16b, v0.16b, 4
1368
	and	v0.16b, v0.16b, v31.16b
1369
	tbl	v0.16b, {v28.16b}, v0.16b
1370
	tbl	v2.16b, {v27.16b}, v2.16b
1371
	eor	v0.16b, v0.16b, v2.16b
1372
	eor	v1.16b, v1.16b, v1.16b
1373
	aese	v0.16b,v1.16b
1374
	ushr	v2.16b, v0.16b, 4
1375
	and	v0.16b, v0.16b, v31.16b
1376
	tbl	v0.16b, {v30.16b}, v0.16b
1377
	tbl	v2.16b, {v29.16b}, v2.16b
1378
	eor	v0.16b, v0.16b, v2.16b
1379

1380
	mov	w7,v0.s[0]
1381
	eor	w6,w7,w7,ror #32-2
1382
	eor	w6,w6,w7,ror #32-10
1383
	eor	w6,w6,w7,ror #32-18
1384
	eor	w6,w6,w7,ror #32-24
1385
	ldp	w7,w8,[x10],8
1386
	eor	w13,w13,w6
1387
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
1388
	eor	w6,w12,w13
1389
	eor	w9,w7,w15
1390
	eor	w6,w6,w9
1391
	mov	v3.s[0],w6
1392
	// optimize sbox using AESE instruction
1393
	tbl	v0.16b, {v3.16b}, v26.16b
1394
	ushr	v2.16b, v0.16b, 4
1395
	and	v0.16b, v0.16b, v31.16b
1396
	tbl	v0.16b, {v28.16b}, v0.16b
1397
	tbl	v2.16b, {v27.16b}, v2.16b
1398
	eor	v0.16b, v0.16b, v2.16b
1399
	eor	v1.16b, v1.16b, v1.16b
1400
	aese	v0.16b,v1.16b
1401
	ushr	v2.16b, v0.16b, 4
1402
	and	v0.16b, v0.16b, v31.16b
1403
	tbl	v0.16b, {v30.16b}, v0.16b
1404
	tbl	v2.16b, {v29.16b}, v2.16b
1405
	eor	v0.16b, v0.16b, v2.16b
1406

1407
	mov	w7,v0.s[0]
1408
	eor	w6,w7,w7,ror #32-2
1409
	eor	w6,w6,w7,ror #32-10
1410
	eor	w6,w6,w7,ror #32-18
1411
	eor	w6,w6,w7,ror #32-24
1412
	eor	w14,w14,w6
1413
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
1414
	eor	w6,w12,w13
1415
	eor	w9,w14,w8
1416
	eor	w6,w6,w9
1417
	mov	v3.s[0],w6
1418
	// optimize sbox using AESE instruction
1419
	tbl	v0.16b, {v3.16b}, v26.16b
1420
	ushr	v2.16b, v0.16b, 4
1421
	and	v0.16b, v0.16b, v31.16b
1422
	tbl	v0.16b, {v28.16b}, v0.16b
1423
	tbl	v2.16b, {v27.16b}, v2.16b
1424
	eor	v0.16b, v0.16b, v2.16b
1425
	eor	v1.16b, v1.16b, v1.16b
1426
	aese	v0.16b,v1.16b
1427
	ushr	v2.16b, v0.16b, 4
1428
	and	v0.16b, v0.16b, v31.16b
1429
	tbl	v0.16b, {v30.16b}, v0.16b
1430
	tbl	v2.16b, {v29.16b}, v2.16b
1431
	eor	v0.16b, v0.16b, v2.16b
1432

1433
	mov	w7,v0.s[0]
1434
	eor	w6,w7,w7,ror #32-2
1435
	eor	w6,w6,w7,ror #32-10
1436
	eor	w6,w6,w7,ror #32-18
1437
	eor	w6,w6,w7,ror #32-24
1438
	eor	w15,w15,w6
1439
	subs	w11,w11,#1
1440
	b.ne	10b
1441
	mov	v5.s[0],w15
1442
	mov	v5.s[1],w14
1443
	mov	v5.s[2],w13
1444
	mov	v5.s[3],w12
1445
#ifndef __AARCH64EB__
1446
	rev32	v4.16b,v4.16b
1447
#endif
1448
	eor	v6.16b,v6.16b,v5.16b
1449
	mov	x10,x3
1450
	mov	w11,#8
1451
	mov	w12,v6.s[0]
1452
	mov	w13,v6.s[1]
1453
	mov	w14,v6.s[2]
1454
	mov	w15,v6.s[3]
1455
10:
1456
	ldp	w7,w8,[x10],8
1457
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
1458
	eor	w6,w14,w15
1459
	eor	w9,w7,w13
1460
	eor	w6,w6,w9
1461
	mov	v3.s[0],w6
1462
	// optimize sbox using AESE instruction
1463
	tbl	v0.16b, {v3.16b}, v26.16b
1464
	ushr	v2.16b, v0.16b, 4
1465
	and	v0.16b, v0.16b, v31.16b
1466
	tbl	v0.16b, {v28.16b}, v0.16b
1467
	tbl	v2.16b, {v27.16b}, v2.16b
1468
	eor	v0.16b, v0.16b, v2.16b
1469
	eor	v1.16b, v1.16b, v1.16b
1470
	aese	v0.16b,v1.16b
1471
	ushr	v2.16b, v0.16b, 4
1472
	and	v0.16b, v0.16b, v31.16b
1473
	tbl	v0.16b, {v30.16b}, v0.16b
1474
	tbl	v2.16b, {v29.16b}, v2.16b
1475
	eor	v0.16b, v0.16b, v2.16b
1476

1477
	mov	w7,v0.s[0]
1478
	eor	w6,w7,w7,ror #32-2
1479
	eor	w6,w6,w7,ror #32-10
1480
	eor	w6,w6,w7,ror #32-18
1481
	eor	w6,w6,w7,ror #32-24
1482
	eor	w12,w12,w6
1483
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
1484
	eor	w6,w14,w15
1485
	eor	w9,w12,w8
1486
	eor	w6,w6,w9
1487
	mov	v3.s[0],w6
1488
	// optimize sbox using AESE instruction
1489
	tbl	v0.16b, {v3.16b}, v26.16b
1490
	ushr	v2.16b, v0.16b, 4
1491
	and	v0.16b, v0.16b, v31.16b
1492
	tbl	v0.16b, {v28.16b}, v0.16b
1493
	tbl	v2.16b, {v27.16b}, v2.16b
1494
	eor	v0.16b, v0.16b, v2.16b
1495
	eor	v1.16b, v1.16b, v1.16b
1496
	aese	v0.16b,v1.16b
1497
	ushr	v2.16b, v0.16b, 4
1498
	and	v0.16b, v0.16b, v31.16b
1499
	tbl	v0.16b, {v30.16b}, v0.16b
1500
	tbl	v2.16b, {v29.16b}, v2.16b
1501
	eor	v0.16b, v0.16b, v2.16b
1502

1503
	mov	w7,v0.s[0]
1504
	eor	w6,w7,w7,ror #32-2
1505
	eor	w6,w6,w7,ror #32-10
1506
	eor	w6,w6,w7,ror #32-18
1507
	eor	w6,w6,w7,ror #32-24
1508
	ldp	w7,w8,[x10],8
1509
	eor	w13,w13,w6
1510
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
1511
	eor	w6,w12,w13
1512
	eor	w9,w7,w15
1513
	eor	w6,w6,w9
1514
	mov	v3.s[0],w6
1515
	// optimize sbox using AESE instruction
1516
	tbl	v0.16b, {v3.16b}, v26.16b
1517
	ushr	v2.16b, v0.16b, 4
1518
	and	v0.16b, v0.16b, v31.16b
1519
	tbl	v0.16b, {v28.16b}, v0.16b
1520
	tbl	v2.16b, {v27.16b}, v2.16b
1521
	eor	v0.16b, v0.16b, v2.16b
1522
	eor	v1.16b, v1.16b, v1.16b
1523
	aese	v0.16b,v1.16b
1524
	ushr	v2.16b, v0.16b, 4
1525
	and	v0.16b, v0.16b, v31.16b
1526
	tbl	v0.16b, {v30.16b}, v0.16b
1527
	tbl	v2.16b, {v29.16b}, v2.16b
1528
	eor	v0.16b, v0.16b, v2.16b
1529

1530
	mov	w7,v0.s[0]
1531
	eor	w6,w7,w7,ror #32-2
1532
	eor	w6,w6,w7,ror #32-10
1533
	eor	w6,w6,w7,ror #32-18
1534
	eor	w6,w6,w7,ror #32-24
1535
	eor	w14,w14,w6
1536
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
1537
	eor	w6,w12,w13
1538
	eor	w9,w14,w8
1539
	eor	w6,w6,w9
1540
	mov	v3.s[0],w6
1541
	// optimize sbox using AESE instruction
1542
	tbl	v0.16b, {v3.16b}, v26.16b
1543
	ushr	v2.16b, v0.16b, 4
1544
	and	v0.16b, v0.16b, v31.16b
1545
	tbl	v0.16b, {v28.16b}, v0.16b
1546
	tbl	v2.16b, {v27.16b}, v2.16b
1547
	eor	v0.16b, v0.16b, v2.16b
1548
	eor	v1.16b, v1.16b, v1.16b
1549
	aese	v0.16b,v1.16b
1550
	ushr	v2.16b, v0.16b, 4
1551
	and	v0.16b, v0.16b, v31.16b
1552
	tbl	v0.16b, {v30.16b}, v0.16b
1553
	tbl	v2.16b, {v29.16b}, v2.16b
1554
	eor	v0.16b, v0.16b, v2.16b
1555

1556
	mov	w7,v0.s[0]
1557
	eor	w6,w7,w7,ror #32-2
1558
	eor	w6,w6,w7,ror #32-10
1559
	eor	w6,w6,w7,ror #32-18
1560
	eor	w6,w6,w7,ror #32-24
1561
	eor	w15,w15,w6
1562
	subs	w11,w11,#1
1563
	b.ne	10b
1564
	mov	v6.s[0],w15
1565
	mov	v6.s[1],w14
1566
	mov	v6.s[2],w13
1567
	mov	v6.s[3],w12
1568
#ifndef __AARCH64EB__
1569
	rev32	v5.16b,v5.16b
1570
#endif
1571
	eor	v7.16b,v7.16b,v6.16b
1572
	mov	x10,x3
1573
	mov	w11,#8
1574
	mov	w12,v7.s[0]
1575
	mov	w13,v7.s[1]
1576
	mov	w14,v7.s[2]
1577
	mov	w15,v7.s[3]
1578
10:
1579
	ldp	w7,w8,[x10],8
1580
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
1581
	eor	w6,w14,w15
1582
	eor	w9,w7,w13
1583
	eor	w6,w6,w9
1584
	mov	v3.s[0],w6
1585
	// optimize sbox using AESE instruction
1586
	tbl	v0.16b, {v3.16b}, v26.16b
1587
	ushr	v2.16b, v0.16b, 4
1588
	and	v0.16b, v0.16b, v31.16b
1589
	tbl	v0.16b, {v28.16b}, v0.16b
1590
	tbl	v2.16b, {v27.16b}, v2.16b
1591
	eor	v0.16b, v0.16b, v2.16b
1592
	eor	v1.16b, v1.16b, v1.16b
1593
	aese	v0.16b,v1.16b
1594
	ushr	v2.16b, v0.16b, 4
1595
	and	v0.16b, v0.16b, v31.16b
1596
	tbl	v0.16b, {v30.16b}, v0.16b
1597
	tbl	v2.16b, {v29.16b}, v2.16b
1598
	eor	v0.16b, v0.16b, v2.16b
1599

1600
	mov	w7,v0.s[0]
1601
	eor	w6,w7,w7,ror #32-2
1602
	eor	w6,w6,w7,ror #32-10
1603
	eor	w6,w6,w7,ror #32-18
1604
	eor	w6,w6,w7,ror #32-24
1605
	eor	w12,w12,w6
1606
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
1607
	eor	w6,w14,w15
1608
	eor	w9,w12,w8
1609
	eor	w6,w6,w9
1610
	mov	v3.s[0],w6
1611
	// optimize sbox using AESE instruction
1612
	tbl	v0.16b, {v3.16b}, v26.16b
1613
	ushr	v2.16b, v0.16b, 4
1614
	and	v0.16b, v0.16b, v31.16b
1615
	tbl	v0.16b, {v28.16b}, v0.16b
1616
	tbl	v2.16b, {v27.16b}, v2.16b
1617
	eor	v0.16b, v0.16b, v2.16b
1618
	eor	v1.16b, v1.16b, v1.16b
1619
	aese	v0.16b,v1.16b
1620
	ushr	v2.16b, v0.16b, 4
1621
	and	v0.16b, v0.16b, v31.16b
1622
	tbl	v0.16b, {v30.16b}, v0.16b
1623
	tbl	v2.16b, {v29.16b}, v2.16b
1624
	eor	v0.16b, v0.16b, v2.16b
1625

1626
	mov	w7,v0.s[0]
1627
	eor	w6,w7,w7,ror #32-2
1628
	eor	w6,w6,w7,ror #32-10
1629
	eor	w6,w6,w7,ror #32-18
1630
	eor	w6,w6,w7,ror #32-24
1631
	ldp	w7,w8,[x10],8
1632
	eor	w13,w13,w6
1633
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
1634
	eor	w6,w12,w13
1635
	eor	w9,w7,w15
1636
	eor	w6,w6,w9
1637
	mov	v3.s[0],w6
1638
	// optimize sbox using AESE instruction
1639
	tbl	v0.16b, {v3.16b}, v26.16b
1640
	ushr	v2.16b, v0.16b, 4
1641
	and	v0.16b, v0.16b, v31.16b
1642
	tbl	v0.16b, {v28.16b}, v0.16b
1643
	tbl	v2.16b, {v27.16b}, v2.16b
1644
	eor	v0.16b, v0.16b, v2.16b
1645
	eor	v1.16b, v1.16b, v1.16b
1646
	aese	v0.16b,v1.16b
1647
	ushr	v2.16b, v0.16b, 4
1648
	and	v0.16b, v0.16b, v31.16b
1649
	tbl	v0.16b, {v30.16b}, v0.16b
1650
	tbl	v2.16b, {v29.16b}, v2.16b
1651
	eor	v0.16b, v0.16b, v2.16b
1652

1653
	mov	w7,v0.s[0]
1654
	eor	w6,w7,w7,ror #32-2
1655
	eor	w6,w6,w7,ror #32-10
1656
	eor	w6,w6,w7,ror #32-18
1657
	eor	w6,w6,w7,ror #32-24
1658
	eor	w14,w14,w6
1659
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
1660
	eor	w6,w12,w13
1661
	eor	w9,w14,w8
1662
	eor	w6,w6,w9
1663
	mov	v3.s[0],w6
1664
	// optimize sbox using AESE instruction
1665
	tbl	v0.16b, {v3.16b}, v26.16b
1666
	ushr	v2.16b, v0.16b, 4
1667
	and	v0.16b, v0.16b, v31.16b
1668
	tbl	v0.16b, {v28.16b}, v0.16b
1669
	tbl	v2.16b, {v27.16b}, v2.16b
1670
	eor	v0.16b, v0.16b, v2.16b
1671
	eor	v1.16b, v1.16b, v1.16b
1672
	aese	v0.16b,v1.16b
1673
	ushr	v2.16b, v0.16b, 4
1674
	and	v0.16b, v0.16b, v31.16b
1675
	tbl	v0.16b, {v30.16b}, v0.16b
1676
	tbl	v2.16b, {v29.16b}, v2.16b
1677
	eor	v0.16b, v0.16b, v2.16b
1678

1679
	mov	w7,v0.s[0]
1680
	eor	w6,w7,w7,ror #32-2
1681
	eor	w6,w6,w7,ror #32-10
1682
	eor	w6,w6,w7,ror #32-18
1683
	eor	w6,w6,w7,ror #32-24
1684
	eor	w15,w15,w6
1685
	subs	w11,w11,#1
1686
	b.ne	10b
1687
	mov	v7.s[0],w15
1688
	mov	v7.s[1],w14
1689
	mov	v7.s[2],w13
1690
	mov	v7.s[3],w12
1691
#ifndef __AARCH64EB__
1692
	rev32	v6.16b,v6.16b
1693
#endif
1694
#ifndef __AARCH64EB__
1695
	rev32	v7.16b,v7.16b
1696
#endif
1697
	orr	v3.16b,v7.16b,v7.16b
1698
	st1	{v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64
1699
	subs	w2,w2,#4
1700
	b.ne	.Lcbc_4_blocks_enc
1701
	b	2f
1702
1:
1703
	subs	w2,w2,#1
1704
	b.lt	2f
1705
	ld1	{v4.4s},[x0],#16
1706
	eor	v3.16b,v3.16b,v4.16b
1707
#ifndef __AARCH64EB__
1708
	rev32	v3.16b,v3.16b
1709
#endif
1710
	mov	x10,x3
1711
	mov	w11,#8
1712
	mov	w12,v3.s[0]
1713
	mov	w13,v3.s[1]
1714
	mov	w14,v3.s[2]
1715
	mov	w15,v3.s[3]
1716
10:
1717
	ldp	w7,w8,[x10],8
1718
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
1719
	eor	w6,w14,w15
1720
	eor	w9,w7,w13
1721
	eor	w6,w6,w9
1722
	mov	v3.s[0],w6
1723
	// optimize sbox using AESE instruction
1724
	tbl	v0.16b, {v3.16b}, v26.16b
1725
	ushr	v2.16b, v0.16b, 4
1726
	and	v0.16b, v0.16b, v31.16b
1727
	tbl	v0.16b, {v28.16b}, v0.16b
1728
	tbl	v2.16b, {v27.16b}, v2.16b
1729
	eor	v0.16b, v0.16b, v2.16b
1730
	eor	v1.16b, v1.16b, v1.16b
1731
	aese	v0.16b,v1.16b
1732
	ushr	v2.16b, v0.16b, 4
1733
	and	v0.16b, v0.16b, v31.16b
1734
	tbl	v0.16b, {v30.16b}, v0.16b
1735
	tbl	v2.16b, {v29.16b}, v2.16b
1736
	eor	v0.16b, v0.16b, v2.16b
1737

1738
	mov	w7,v0.s[0]
1739
	eor	w6,w7,w7,ror #32-2
1740
	eor	w6,w6,w7,ror #32-10
1741
	eor	w6,w6,w7,ror #32-18
1742
	eor	w6,w6,w7,ror #32-24
1743
	eor	w12,w12,w6
1744
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
1745
	eor	w6,w14,w15
1746
	eor	w9,w12,w8
1747
	eor	w6,w6,w9
1748
	mov	v3.s[0],w6
1749
	// optimize sbox using AESE instruction
1750
	tbl	v0.16b, {v3.16b}, v26.16b
1751
	ushr	v2.16b, v0.16b, 4
1752
	and	v0.16b, v0.16b, v31.16b
1753
	tbl	v0.16b, {v28.16b}, v0.16b
1754
	tbl	v2.16b, {v27.16b}, v2.16b
1755
	eor	v0.16b, v0.16b, v2.16b
1756
	eor	v1.16b, v1.16b, v1.16b
1757
	aese	v0.16b,v1.16b
1758
	ushr	v2.16b, v0.16b, 4
1759
	and	v0.16b, v0.16b, v31.16b
1760
	tbl	v0.16b, {v30.16b}, v0.16b
1761
	tbl	v2.16b, {v29.16b}, v2.16b
1762
	eor	v0.16b, v0.16b, v2.16b
1763

1764
	mov	w7,v0.s[0]
1765
	eor	w6,w7,w7,ror #32-2
1766
	eor	w6,w6,w7,ror #32-10
1767
	eor	w6,w6,w7,ror #32-18
1768
	eor	w6,w6,w7,ror #32-24
1769
	ldp	w7,w8,[x10],8
1770
	eor	w13,w13,w6
1771
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
1772
	eor	w6,w12,w13
1773
	eor	w9,w7,w15
1774
	eor	w6,w6,w9
1775
	mov	v3.s[0],w6
1776
	// optimize sbox using AESE instruction
1777
	tbl	v0.16b, {v3.16b}, v26.16b
1778
	ushr	v2.16b, v0.16b, 4
1779
	and	v0.16b, v0.16b, v31.16b
1780
	tbl	v0.16b, {v28.16b}, v0.16b
1781
	tbl	v2.16b, {v27.16b}, v2.16b
1782
	eor	v0.16b, v0.16b, v2.16b
1783
	eor	v1.16b, v1.16b, v1.16b
1784
	aese	v0.16b,v1.16b
1785
	ushr	v2.16b, v0.16b, 4
1786
	and	v0.16b, v0.16b, v31.16b
1787
	tbl	v0.16b, {v30.16b}, v0.16b
1788
	tbl	v2.16b, {v29.16b}, v2.16b
1789
	eor	v0.16b, v0.16b, v2.16b
1790

1791
	mov	w7,v0.s[0]
1792
	eor	w6,w7,w7,ror #32-2
1793
	eor	w6,w6,w7,ror #32-10
1794
	eor	w6,w6,w7,ror #32-18
1795
	eor	w6,w6,w7,ror #32-24
1796
	eor	w14,w14,w6
1797
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
1798
	eor	w6,w12,w13
1799
	eor	w9,w14,w8
1800
	eor	w6,w6,w9
1801
	mov	v3.s[0],w6
1802
	// optimize sbox using AESE instruction
1803
	tbl	v0.16b, {v3.16b}, v26.16b
1804
	ushr	v2.16b, v0.16b, 4
1805
	and	v0.16b, v0.16b, v31.16b
1806
	tbl	v0.16b, {v28.16b}, v0.16b
1807
	tbl	v2.16b, {v27.16b}, v2.16b
1808
	eor	v0.16b, v0.16b, v2.16b
1809
	eor	v1.16b, v1.16b, v1.16b
1810
	aese	v0.16b,v1.16b
1811
	ushr	v2.16b, v0.16b, 4
1812
	and	v0.16b, v0.16b, v31.16b
1813
	tbl	v0.16b, {v30.16b}, v0.16b
1814
	tbl	v2.16b, {v29.16b}, v2.16b
1815
	eor	v0.16b, v0.16b, v2.16b
1816

1817
	mov	w7,v0.s[0]
1818
	eor	w6,w7,w7,ror #32-2
1819
	eor	w6,w6,w7,ror #32-10
1820
	eor	w6,w6,w7,ror #32-18
1821
	eor	w6,w6,w7,ror #32-24
1822
	eor	w15,w15,w6
1823
	subs	w11,w11,#1
1824
	b.ne	10b
1825
	mov	v3.s[0],w15
1826
	mov	v3.s[1],w14
1827
	mov	v3.s[2],w13
1828
	mov	v3.s[3],w12
1829
#ifndef __AARCH64EB__
1830
	rev32	v3.16b,v3.16b
1831
#endif
1832
	st1	{v3.4s},[x1],#16
1833
	b	1b
1834
2:
1835
	// save back IV
1836
	st1	{v3.4s},[x4]
1837
	ret
1838

1839
.Ldec:
1840
	// decryption mode starts
1841
	AARCH64_SIGN_LINK_REGISTER
1842
	stp	d8,d9,[sp,#-80]!
1843
	stp	d10,d11,[sp,#16]
1844
	stp	d12,d13,[sp,#32]
1845
	stp	d14,d15,[sp,#48]
1846
	stp	x29,x30,[sp,#64]
1847
.Lcbc_8_blocks_dec:
1848
	cmp	w2,#8
1849
	b.lt	1f
1850
	ld4	{v4.4s,v5.4s,v6.4s,v7.4s},[x0]
1851
	add	x10,x0,#64
1852
	ld4	{v8.4s,v9.4s,v10.4s,v11.4s},[x10]
1853
#ifndef __AARCH64EB__
1854
	rev32	v4.16b,v4.16b
1855
#endif
1856
#ifndef __AARCH64EB__
1857
	rev32	v5.16b,v5.16b
1858
#endif
1859
#ifndef __AARCH64EB__
1860
	rev32	v6.16b,v6.16b
1861
#endif
1862
#ifndef __AARCH64EB__
1863
	rev32	v7.16b,v7.16b
1864
#endif
1865
#ifndef __AARCH64EB__
1866
	rev32	v8.16b,v8.16b
1867
#endif
1868
#ifndef __AARCH64EB__
1869
	rev32	v9.16b,v9.16b
1870
#endif
1871
#ifndef __AARCH64EB__
1872
	rev32	v10.16b,v10.16b
1873
#endif
1874
#ifndef __AARCH64EB__
1875
	rev32	v11.16b,v11.16b
1876
#endif
1877
	bl	_vpsm4_ex_enc_8blks
1878
	zip1	v8.4s,v0.4s,v1.4s
1879
	zip2	v9.4s,v0.4s,v1.4s
1880
	zip1	v10.4s,v2.4s,v3.4s
1881
	zip2	v11.4s,v2.4s,v3.4s
1882
	zip1	v0.2d,v8.2d,v10.2d
1883
	zip2	v1.2d,v8.2d,v10.2d
1884
	zip1	v2.2d,v9.2d,v11.2d
1885
	zip2	v3.2d,v9.2d,v11.2d
1886
	zip1	v8.4s,v4.4s,v5.4s
1887
	zip2	v9.4s,v4.4s,v5.4s
1888
	zip1	v10.4s,v6.4s,v7.4s
1889
	zip2	v11.4s,v6.4s,v7.4s
1890
	zip1	v4.2d,v8.2d,v10.2d
1891
	zip2	v5.2d,v8.2d,v10.2d
1892
	zip1	v6.2d,v9.2d,v11.2d
1893
	zip2	v7.2d,v9.2d,v11.2d
1894
	ld1	{v15.4s},[x4]
1895
	ld1	{v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64
1896
	// note ivec1 and vtmpx[3] are reusing the same register
1897
	// care needs to be taken to avoid conflict
1898
	eor	v0.16b,v0.16b,v15.16b
1899
	ld1	{v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64
1900
	eor	v1.16b,v1.16b,v8.16b
1901
	eor	v2.16b,v2.16b,v9.16b
1902
	eor	v3.16b,v3.16b,v10.16b
1903
	// save back IV
1904
	st1	{v15.4s}, [x4]
1905
	eor	v4.16b,v4.16b,v11.16b
1906
	eor	v5.16b,v5.16b,v12.16b
1907
	eor	v6.16b,v6.16b,v13.16b
1908
	eor	v7.16b,v7.16b,v14.16b
1909
	st1	{v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64
1910
	st1	{v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64
1911
	subs	w2,w2,#8
1912
	b.gt	.Lcbc_8_blocks_dec
1913
	b.eq	100f
1914
1:
1915
	ld1	{v15.4s},[x4]
1916
.Lcbc_4_blocks_dec:
1917
	cmp	w2,#4
1918
	b.lt	1f
1919
	ld4	{v4.4s,v5.4s,v6.4s,v7.4s},[x0]
1920
#ifndef __AARCH64EB__
1921
	rev32	v4.16b,v4.16b
1922
#endif
1923
#ifndef __AARCH64EB__
1924
	rev32	v5.16b,v5.16b
1925
#endif
1926
#ifndef __AARCH64EB__
1927
	rev32	v6.16b,v6.16b
1928
#endif
1929
#ifndef __AARCH64EB__
1930
	rev32	v7.16b,v7.16b
1931
#endif
1932
	bl	_vpsm4_ex_enc_4blks
1933
	ld1	{v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64
1934
	zip1	v8.4s,v0.4s,v1.4s
1935
	zip2	v9.4s,v0.4s,v1.4s
1936
	zip1	v10.4s,v2.4s,v3.4s
1937
	zip2	v11.4s,v2.4s,v3.4s
1938
	zip1	v0.2d,v8.2d,v10.2d
1939
	zip2	v1.2d,v8.2d,v10.2d
1940
	zip1	v2.2d,v9.2d,v11.2d
1941
	zip2	v3.2d,v9.2d,v11.2d
1942
	eor	v0.16b,v0.16b,v15.16b
1943
	eor	v1.16b,v1.16b,v4.16b
1944
	orr	v15.16b,v7.16b,v7.16b
1945
	eor	v2.16b,v2.16b,v5.16b
1946
	eor	v3.16b,v3.16b,v6.16b
1947
	st1	{v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64
1948
	subs	w2,w2,#4
1949
	b.gt	.Lcbc_4_blocks_dec
1950
	// save back IV
1951
	st1	{v7.4s}, [x4]
1952
	b	100f
1953
1:	//	last block
1954
	subs	w2,w2,#1
1955
	b.lt	100f
1956
	b.gt	1f
1957
	ld1	{v4.4s},[x0],#16
1958
	// save back IV
1959
	st1	{v4.4s}, [x4]
1960
#ifndef __AARCH64EB__
1961
	rev32	v8.16b,v4.16b
1962
#else
1963
	mov	v8.16b,v4.16b
1964
#endif
1965
	mov	x10,x3
1966
	mov	w11,#8
1967
	mov	w12,v8.s[0]
1968
	mov	w13,v8.s[1]
1969
	mov	w14,v8.s[2]
1970
	mov	w15,v8.s[3]
1971
10:
1972
	ldp	w7,w8,[x10],8
1973
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
1974
	eor	w6,w14,w15
1975
	eor	w9,w7,w13
1976
	eor	w6,w6,w9
1977
	mov	v3.s[0],w6
1978
	// optimize sbox using AESE instruction
1979
	tbl	v0.16b, {v3.16b}, v26.16b
1980
	ushr	v2.16b, v0.16b, 4
1981
	and	v0.16b, v0.16b, v31.16b
1982
	tbl	v0.16b, {v28.16b}, v0.16b
1983
	tbl	v2.16b, {v27.16b}, v2.16b
1984
	eor	v0.16b, v0.16b, v2.16b
1985
	eor	v1.16b, v1.16b, v1.16b
1986
	aese	v0.16b,v1.16b
1987
	ushr	v2.16b, v0.16b, 4
1988
	and	v0.16b, v0.16b, v31.16b
1989
	tbl	v0.16b, {v30.16b}, v0.16b
1990
	tbl	v2.16b, {v29.16b}, v2.16b
1991
	eor	v0.16b, v0.16b, v2.16b
1992

1993
	mov	w7,v0.s[0]
1994
	eor	w6,w7,w7,ror #32-2
1995
	eor	w6,w6,w7,ror #32-10
1996
	eor	w6,w6,w7,ror #32-18
1997
	eor	w6,w6,w7,ror #32-24
1998
	eor	w12,w12,w6
1999
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
2000
	eor	w6,w14,w15
2001
	eor	w9,w12,w8
2002
	eor	w6,w6,w9
2003
	mov	v3.s[0],w6
2004
	// optimize sbox using AESE instruction
2005
	tbl	v0.16b, {v3.16b}, v26.16b
2006
	ushr	v2.16b, v0.16b, 4
2007
	and	v0.16b, v0.16b, v31.16b
2008
	tbl	v0.16b, {v28.16b}, v0.16b
2009
	tbl	v2.16b, {v27.16b}, v2.16b
2010
	eor	v0.16b, v0.16b, v2.16b
2011
	eor	v1.16b, v1.16b, v1.16b
2012
	aese	v0.16b,v1.16b
2013
	ushr	v2.16b, v0.16b, 4
2014
	and	v0.16b, v0.16b, v31.16b
2015
	tbl	v0.16b, {v30.16b}, v0.16b
2016
	tbl	v2.16b, {v29.16b}, v2.16b
2017
	eor	v0.16b, v0.16b, v2.16b
2018

2019
	mov	w7,v0.s[0]
2020
	eor	w6,w7,w7,ror #32-2
2021
	eor	w6,w6,w7,ror #32-10
2022
	eor	w6,w6,w7,ror #32-18
2023
	eor	w6,w6,w7,ror #32-24
2024
	ldp	w7,w8,[x10],8
2025
	eor	w13,w13,w6
2026
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
2027
	eor	w6,w12,w13
2028
	eor	w9,w7,w15
2029
	eor	w6,w6,w9
2030
	mov	v3.s[0],w6
2031
	// optimize sbox using AESE instruction
2032
	tbl	v0.16b, {v3.16b}, v26.16b
2033
	ushr	v2.16b, v0.16b, 4
2034
	and	v0.16b, v0.16b, v31.16b
2035
	tbl	v0.16b, {v28.16b}, v0.16b
2036
	tbl	v2.16b, {v27.16b}, v2.16b
2037
	eor	v0.16b, v0.16b, v2.16b
2038
	eor	v1.16b, v1.16b, v1.16b
2039
	aese	v0.16b,v1.16b
2040
	ushr	v2.16b, v0.16b, 4
2041
	and	v0.16b, v0.16b, v31.16b
2042
	tbl	v0.16b, {v30.16b}, v0.16b
2043
	tbl	v2.16b, {v29.16b}, v2.16b
2044
	eor	v0.16b, v0.16b, v2.16b
2045

2046
	mov	w7,v0.s[0]
2047
	eor	w6,w7,w7,ror #32-2
2048
	eor	w6,w6,w7,ror #32-10
2049
	eor	w6,w6,w7,ror #32-18
2050
	eor	w6,w6,w7,ror #32-24
2051
	eor	w14,w14,w6
2052
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
2053
	eor	w6,w12,w13
2054
	eor	w9,w14,w8
2055
	eor	w6,w6,w9
2056
	mov	v3.s[0],w6
2057
	// optimize sbox using AESE instruction
2058
	tbl	v0.16b, {v3.16b}, v26.16b
2059
	ushr	v2.16b, v0.16b, 4
2060
	and	v0.16b, v0.16b, v31.16b
2061
	tbl	v0.16b, {v28.16b}, v0.16b
2062
	tbl	v2.16b, {v27.16b}, v2.16b
2063
	eor	v0.16b, v0.16b, v2.16b
2064
	eor	v1.16b, v1.16b, v1.16b
2065
	aese	v0.16b,v1.16b
2066
	ushr	v2.16b, v0.16b, 4
2067
	and	v0.16b, v0.16b, v31.16b
2068
	tbl	v0.16b, {v30.16b}, v0.16b
2069
	tbl	v2.16b, {v29.16b}, v2.16b
2070
	eor	v0.16b, v0.16b, v2.16b
2071

2072
	mov	w7,v0.s[0]
2073
	eor	w6,w7,w7,ror #32-2
2074
	eor	w6,w6,w7,ror #32-10
2075
	eor	w6,w6,w7,ror #32-18
2076
	eor	w6,w6,w7,ror #32-24
2077
	eor	w15,w15,w6
2078
	subs	w11,w11,#1
2079
	b.ne	10b
2080
	mov	v8.s[0],w15
2081
	mov	v8.s[1],w14
2082
	mov	v8.s[2],w13
2083
	mov	v8.s[3],w12
2084
#ifndef __AARCH64EB__
2085
	rev32	v8.16b,v8.16b
2086
#endif
2087
	eor	v8.16b,v8.16b,v15.16b
2088
	st1	{v8.4s},[x1],#16
2089
	b	100f
2090
1:	//	last two blocks
2091
	ld4	{v4.s,v5.s,v6.s,v7.s}[0],[x0]
2092
	add	x10,x0,#16
2093
	ld4	{v4.s,v5.s,v6.s,v7.s}[1],[x10],#16
2094
	subs	w2,w2,1
2095
	b.gt	1f
2096
#ifndef __AARCH64EB__
2097
	rev32	v4.16b,v4.16b
2098
#endif
2099
#ifndef __AARCH64EB__
2100
	rev32	v5.16b,v5.16b
2101
#endif
2102
#ifndef __AARCH64EB__
2103
	rev32	v6.16b,v6.16b
2104
#endif
2105
#ifndef __AARCH64EB__
2106
	rev32	v7.16b,v7.16b
2107
#endif
2108
	bl	_vpsm4_ex_enc_4blks
2109
	ld1	{v4.4s,v5.4s},[x0],#32
2110
	zip1	v8.4s,v0.4s,v1.4s
2111
	zip2	v9.4s,v0.4s,v1.4s
2112
	zip1	v10.4s,v2.4s,v3.4s
2113
	zip2	v11.4s,v2.4s,v3.4s
2114
	zip1	v0.2d,v8.2d,v10.2d
2115
	zip2	v1.2d,v8.2d,v10.2d
2116
	zip1	v2.2d,v9.2d,v11.2d
2117
	zip2	v3.2d,v9.2d,v11.2d
2118
	eor	v0.16b,v0.16b,v15.16b
2119
	eor	v1.16b,v1.16b,v4.16b
2120
	st1	{v0.4s,v1.4s},[x1],#32
2121
	// save back IV
2122
	st1	{v5.4s}, [x4]
2123
	b	100f
2124
1:	//	last 3 blocks
2125
	ld4	{v4.s,v5.s,v6.s,v7.s}[2],[x10]
2126
#ifndef __AARCH64EB__
2127
	rev32	v4.16b,v4.16b
2128
#endif
2129
#ifndef __AARCH64EB__
2130
	rev32	v5.16b,v5.16b
2131
#endif
2132
#ifndef __AARCH64EB__
2133
	rev32	v6.16b,v6.16b
2134
#endif
2135
#ifndef __AARCH64EB__
2136
	rev32	v7.16b,v7.16b
2137
#endif
2138
	bl	_vpsm4_ex_enc_4blks
2139
	ld1	{v4.4s,v5.4s,v6.4s},[x0],#48
2140
	zip1	v8.4s,v0.4s,v1.4s
2141
	zip2	v9.4s,v0.4s,v1.4s
2142
	zip1	v10.4s,v2.4s,v3.4s
2143
	zip2	v11.4s,v2.4s,v3.4s
2144
	zip1	v0.2d,v8.2d,v10.2d
2145
	zip2	v1.2d,v8.2d,v10.2d
2146
	zip1	v2.2d,v9.2d,v11.2d
2147
	zip2	v3.2d,v9.2d,v11.2d
2148
	eor	v0.16b,v0.16b,v15.16b
2149
	eor	v1.16b,v1.16b,v4.16b
2150
	eor	v2.16b,v2.16b,v5.16b
2151
	st1	{v0.4s,v1.4s,v2.4s},[x1],#48
2152
	// save back IV
2153
	st1	{v6.4s}, [x4]
2154
100:
2155
	ldp	d10,d11,[sp,#16]
2156
	ldp	d12,d13,[sp,#32]
2157
	ldp	d14,d15,[sp,#48]
2158
	ldp	x29,x30,[sp,#64]
2159
	ldp	d8,d9,[sp],#80
2160
	AARCH64_VALIDATE_LINK_REGISTER
2161
	ret
2162
.size	vpsm4_ex_cbc_encrypt,.-vpsm4_ex_cbc_encrypt
2163
.globl	vpsm4_ex_ctr32_encrypt_blocks
2164
.type	vpsm4_ex_ctr32_encrypt_blocks,%function
2165
.align	5
2166
vpsm4_ex_ctr32_encrypt_blocks:
2167
	AARCH64_VALID_CALL_TARGET
2168
	ld1	{v3.4s},[x4]
2169
#ifndef __AARCH64EB__
2170
	rev32	v3.16b,v3.16b
2171
#endif
2172
	adrp	x9, .Lsbox_magic
2173
	ldr	q26, [x9, #:lo12:.Lsbox_magic]
2174
	ldr	q27, [x9, #:lo12:.Lsbox_magic+16]
2175
	ldr	q28, [x9, #:lo12:.Lsbox_magic+32]
2176
	ldr	q29, [x9, #:lo12:.Lsbox_magic+48]
2177
	ldr	q30, [x9, #:lo12:.Lsbox_magic+64]
2178
	ldr	q31, [x9, #:lo12:.Lsbox_magic+80]
2179
	cmp	w2,#1
2180
	b.ne	1f
2181
	// fast processing for one single block without
2182
	// context saving overhead
2183
	mov	x10,x3
2184
	mov	w11,#8
2185
	mov	w12,v3.s[0]
2186
	mov	w13,v3.s[1]
2187
	mov	w14,v3.s[2]
2188
	mov	w15,v3.s[3]
2189
10:
2190
	ldp	w7,w8,[x10],8
2191
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
2192
	eor	w6,w14,w15
2193
	eor	w9,w7,w13
2194
	eor	w6,w6,w9
2195
	mov	v3.s[0],w6
2196
	// optimize sbox using AESE instruction
2197
	tbl	v0.16b, {v3.16b}, v26.16b
2198
	ushr	v2.16b, v0.16b, 4
2199
	and	v0.16b, v0.16b, v31.16b
2200
	tbl	v0.16b, {v28.16b}, v0.16b
2201
	tbl	v2.16b, {v27.16b}, v2.16b
2202
	eor	v0.16b, v0.16b, v2.16b
2203
	eor	v1.16b, v1.16b, v1.16b
2204
	aese	v0.16b,v1.16b
2205
	ushr	v2.16b, v0.16b, 4
2206
	and	v0.16b, v0.16b, v31.16b
2207
	tbl	v0.16b, {v30.16b}, v0.16b
2208
	tbl	v2.16b, {v29.16b}, v2.16b
2209
	eor	v0.16b, v0.16b, v2.16b
2210

2211
	mov	w7,v0.s[0]
2212
	eor	w6,w7,w7,ror #32-2
2213
	eor	w6,w6,w7,ror #32-10
2214
	eor	w6,w6,w7,ror #32-18
2215
	eor	w6,w6,w7,ror #32-24
2216
	eor	w12,w12,w6
2217
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
2218
	eor	w6,w14,w15
2219
	eor	w9,w12,w8
2220
	eor	w6,w6,w9
2221
	mov	v3.s[0],w6
2222
	// optimize sbox using AESE instruction
2223
	tbl	v0.16b, {v3.16b}, v26.16b
2224
	ushr	v2.16b, v0.16b, 4
2225
	and	v0.16b, v0.16b, v31.16b
2226
	tbl	v0.16b, {v28.16b}, v0.16b
2227
	tbl	v2.16b, {v27.16b}, v2.16b
2228
	eor	v0.16b, v0.16b, v2.16b
2229
	eor	v1.16b, v1.16b, v1.16b
2230
	aese	v0.16b,v1.16b
2231
	ushr	v2.16b, v0.16b, 4
2232
	and	v0.16b, v0.16b, v31.16b
2233
	tbl	v0.16b, {v30.16b}, v0.16b
2234
	tbl	v2.16b, {v29.16b}, v2.16b
2235
	eor	v0.16b, v0.16b, v2.16b
2236

2237
	mov	w7,v0.s[0]
2238
	eor	w6,w7,w7,ror #32-2
2239
	eor	w6,w6,w7,ror #32-10
2240
	eor	w6,w6,w7,ror #32-18
2241
	eor	w6,w6,w7,ror #32-24
2242
	ldp	w7,w8,[x10],8
2243
	eor	w13,w13,w6
2244
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
2245
	eor	w6,w12,w13
2246
	eor	w9,w7,w15
2247
	eor	w6,w6,w9
2248
	mov	v3.s[0],w6
2249
	// optimize sbox using AESE instruction
2250
	tbl	v0.16b, {v3.16b}, v26.16b
2251
	ushr	v2.16b, v0.16b, 4
2252
	and	v0.16b, v0.16b, v31.16b
2253
	tbl	v0.16b, {v28.16b}, v0.16b
2254
	tbl	v2.16b, {v27.16b}, v2.16b
2255
	eor	v0.16b, v0.16b, v2.16b
2256
	eor	v1.16b, v1.16b, v1.16b
2257
	aese	v0.16b,v1.16b
2258
	ushr	v2.16b, v0.16b, 4
2259
	and	v0.16b, v0.16b, v31.16b
2260
	tbl	v0.16b, {v30.16b}, v0.16b
2261
	tbl	v2.16b, {v29.16b}, v2.16b
2262
	eor	v0.16b, v0.16b, v2.16b
2263

2264
	mov	w7,v0.s[0]
2265
	eor	w6,w7,w7,ror #32-2
2266
	eor	w6,w6,w7,ror #32-10
2267
	eor	w6,w6,w7,ror #32-18
2268
	eor	w6,w6,w7,ror #32-24
2269
	eor	w14,w14,w6
2270
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
2271
	eor	w6,w12,w13
2272
	eor	w9,w14,w8
2273
	eor	w6,w6,w9
2274
	mov	v3.s[0],w6
2275
	// optimize sbox using AESE instruction
2276
	tbl	v0.16b, {v3.16b}, v26.16b
2277
	ushr	v2.16b, v0.16b, 4
2278
	and	v0.16b, v0.16b, v31.16b
2279
	tbl	v0.16b, {v28.16b}, v0.16b
2280
	tbl	v2.16b, {v27.16b}, v2.16b
2281
	eor	v0.16b, v0.16b, v2.16b
2282
	eor	v1.16b, v1.16b, v1.16b
2283
	aese	v0.16b,v1.16b
2284
	ushr	v2.16b, v0.16b, 4
2285
	and	v0.16b, v0.16b, v31.16b
2286
	tbl	v0.16b, {v30.16b}, v0.16b
2287
	tbl	v2.16b, {v29.16b}, v2.16b
2288
	eor	v0.16b, v0.16b, v2.16b
2289

2290
	mov	w7,v0.s[0]
2291
	eor	w6,w7,w7,ror #32-2
2292
	eor	w6,w6,w7,ror #32-10
2293
	eor	w6,w6,w7,ror #32-18
2294
	eor	w6,w6,w7,ror #32-24
2295
	eor	w15,w15,w6
2296
	subs	w11,w11,#1
2297
	b.ne	10b
2298
	mov	v3.s[0],w15
2299
	mov	v3.s[1],w14
2300
	mov	v3.s[2],w13
2301
	mov	v3.s[3],w12
2302
#ifndef __AARCH64EB__
2303
	rev32	v3.16b,v3.16b
2304
#endif
2305
	ld1	{v4.4s},[x0]
2306
	eor	v4.16b,v4.16b,v3.16b
2307
	st1	{v4.4s},[x1]
2308
	ret
2309
1:
2310
	AARCH64_SIGN_LINK_REGISTER
2311
	stp	d8,d9,[sp,#-80]!
2312
	stp	d10,d11,[sp,#16]
2313
	stp	d12,d13,[sp,#32]
2314
	stp	d14,d15,[sp,#48]
2315
	stp	x29,x30,[sp,#64]
2316
	mov	w12,v3.s[0]
2317
	mov	w13,v3.s[1]
2318
	mov	w14,v3.s[2]
2319
	mov	w5,v3.s[3]
2320
.Lctr32_4_blocks_process:
2321
	cmp	w2,#4
2322
	b.lt	1f
2323
	dup	v4.4s,w12
2324
	dup	v5.4s,w13
2325
	dup	v6.4s,w14
2326
	mov	v7.s[0],w5
2327
	add	w5,w5,#1
2328
	mov	v7.s[1],w5
2329
	add	w5,w5,#1
2330
	mov	v7.s[2],w5
2331
	add	w5,w5,#1
2332
	mov	v7.s[3],w5
2333
	add	w5,w5,#1
2334
	cmp	w2,#8
2335
	b.ge	.Lctr32_8_blocks_process
2336
	bl	_vpsm4_ex_enc_4blks
2337
	ld4	{v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64
2338
	eor	v0.16b,v0.16b,v12.16b
2339
	eor	v1.16b,v1.16b,v13.16b
2340
	eor	v2.16b,v2.16b,v14.16b
2341
	eor	v3.16b,v3.16b,v15.16b
2342
	st4	{v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64
2343
	subs	w2,w2,#4
2344
	b.ne	.Lctr32_4_blocks_process
2345
	b	100f
2346
.Lctr32_8_blocks_process:
2347
	dup	v8.4s,w12
2348
	dup	v9.4s,w13
2349
	dup	v10.4s,w14
2350
	mov	v11.s[0],w5
2351
	add	w5,w5,#1
2352
	mov	v11.s[1],w5
2353
	add	w5,w5,#1
2354
	mov	v11.s[2],w5
2355
	add	w5,w5,#1
2356
	mov	v11.s[3],w5
2357
	add	w5,w5,#1
2358
	bl	_vpsm4_ex_enc_8blks
2359
	ld4	{v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64
2360
	ld4	{v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64
2361
	eor	v0.16b,v0.16b,v12.16b
2362
	eor	v1.16b,v1.16b,v13.16b
2363
	eor	v2.16b,v2.16b,v14.16b
2364
	eor	v3.16b,v3.16b,v15.16b
2365
	eor	v4.16b,v4.16b,v8.16b
2366
	eor	v5.16b,v5.16b,v9.16b
2367
	eor	v6.16b,v6.16b,v10.16b
2368
	eor	v7.16b,v7.16b,v11.16b
2369
	st4	{v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64
2370
	st4	{v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64
2371
	subs	w2,w2,#8
2372
	b.ne	.Lctr32_4_blocks_process
2373
	b	100f
2374
1:	//	last block processing
2375
	subs	w2,w2,#1
2376
	b.lt	100f
2377
	b.gt	1f
2378
	mov	v3.s[0],w12
2379
	mov	v3.s[1],w13
2380
	mov	v3.s[2],w14
2381
	mov	v3.s[3],w5
2382
	mov	x10,x3
2383
	mov	w11,#8
2384
	mov	w12,v3.s[0]
2385
	mov	w13,v3.s[1]
2386
	mov	w14,v3.s[2]
2387
	mov	w15,v3.s[3]
2388
10:
2389
	ldp	w7,w8,[x10],8
2390
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
2391
	eor	w6,w14,w15
2392
	eor	w9,w7,w13
2393
	eor	w6,w6,w9
2394
	mov	v3.s[0],w6
2395
	// optimize sbox using AESE instruction
2396
	tbl	v0.16b, {v3.16b}, v26.16b
2397
	ushr	v2.16b, v0.16b, 4
2398
	and	v0.16b, v0.16b, v31.16b
2399
	tbl	v0.16b, {v28.16b}, v0.16b
2400
	tbl	v2.16b, {v27.16b}, v2.16b
2401
	eor	v0.16b, v0.16b, v2.16b
2402
	eor	v1.16b, v1.16b, v1.16b
2403
	aese	v0.16b,v1.16b
2404
	ushr	v2.16b, v0.16b, 4
2405
	and	v0.16b, v0.16b, v31.16b
2406
	tbl	v0.16b, {v30.16b}, v0.16b
2407
	tbl	v2.16b, {v29.16b}, v2.16b
2408
	eor	v0.16b, v0.16b, v2.16b
2409

2410
	mov	w7,v0.s[0]
2411
	eor	w6,w7,w7,ror #32-2
2412
	eor	w6,w6,w7,ror #32-10
2413
	eor	w6,w6,w7,ror #32-18
2414
	eor	w6,w6,w7,ror #32-24
2415
	eor	w12,w12,w6
2416
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
2417
	eor	w6,w14,w15
2418
	eor	w9,w12,w8
2419
	eor	w6,w6,w9
2420
	mov	v3.s[0],w6
2421
	// optimize sbox using AESE instruction
2422
	tbl	v0.16b, {v3.16b}, v26.16b
2423
	ushr	v2.16b, v0.16b, 4
2424
	and	v0.16b, v0.16b, v31.16b
2425
	tbl	v0.16b, {v28.16b}, v0.16b
2426
	tbl	v2.16b, {v27.16b}, v2.16b
2427
	eor	v0.16b, v0.16b, v2.16b
2428
	eor	v1.16b, v1.16b, v1.16b
2429
	aese	v0.16b,v1.16b
2430
	ushr	v2.16b, v0.16b, 4
2431
	and	v0.16b, v0.16b, v31.16b
2432
	tbl	v0.16b, {v30.16b}, v0.16b
2433
	tbl	v2.16b, {v29.16b}, v2.16b
2434
	eor	v0.16b, v0.16b, v2.16b
2435

2436
	mov	w7,v0.s[0]
2437
	eor	w6,w7,w7,ror #32-2
2438
	eor	w6,w6,w7,ror #32-10
2439
	eor	w6,w6,w7,ror #32-18
2440
	eor	w6,w6,w7,ror #32-24
2441
	ldp	w7,w8,[x10],8
2442
	eor	w13,w13,w6
2443
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
2444
	eor	w6,w12,w13
2445
	eor	w9,w7,w15
2446
	eor	w6,w6,w9
2447
	mov	v3.s[0],w6
2448
	// optimize sbox using AESE instruction
2449
	tbl	v0.16b, {v3.16b}, v26.16b
2450
	ushr	v2.16b, v0.16b, 4
2451
	and	v0.16b, v0.16b, v31.16b
2452
	tbl	v0.16b, {v28.16b}, v0.16b
2453
	tbl	v2.16b, {v27.16b}, v2.16b
2454
	eor	v0.16b, v0.16b, v2.16b
2455
	eor	v1.16b, v1.16b, v1.16b
2456
	aese	v0.16b,v1.16b
2457
	ushr	v2.16b, v0.16b, 4
2458
	and	v0.16b, v0.16b, v31.16b
2459
	tbl	v0.16b, {v30.16b}, v0.16b
2460
	tbl	v2.16b, {v29.16b}, v2.16b
2461
	eor	v0.16b, v0.16b, v2.16b
2462

2463
	mov	w7,v0.s[0]
2464
	eor	w6,w7,w7,ror #32-2
2465
	eor	w6,w6,w7,ror #32-10
2466
	eor	w6,w6,w7,ror #32-18
2467
	eor	w6,w6,w7,ror #32-24
2468
	eor	w14,w14,w6
2469
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
2470
	eor	w6,w12,w13
2471
	eor	w9,w14,w8
2472
	eor	w6,w6,w9
2473
	mov	v3.s[0],w6
2474
	// optimize sbox using AESE instruction
2475
	tbl	v0.16b, {v3.16b}, v26.16b
2476
	ushr	v2.16b, v0.16b, 4
2477
	and	v0.16b, v0.16b, v31.16b
2478
	tbl	v0.16b, {v28.16b}, v0.16b
2479
	tbl	v2.16b, {v27.16b}, v2.16b
2480
	eor	v0.16b, v0.16b, v2.16b
2481
	eor	v1.16b, v1.16b, v1.16b
2482
	aese	v0.16b,v1.16b
2483
	ushr	v2.16b, v0.16b, 4
2484
	and	v0.16b, v0.16b, v31.16b
2485
	tbl	v0.16b, {v30.16b}, v0.16b
2486
	tbl	v2.16b, {v29.16b}, v2.16b
2487
	eor	v0.16b, v0.16b, v2.16b
2488

2489
	mov	w7,v0.s[0]
2490
	eor	w6,w7,w7,ror #32-2
2491
	eor	w6,w6,w7,ror #32-10
2492
	eor	w6,w6,w7,ror #32-18
2493
	eor	w6,w6,w7,ror #32-24
2494
	eor	w15,w15,w6
2495
	subs	w11,w11,#1
2496
	b.ne	10b
2497
	mov	v3.s[0],w15
2498
	mov	v3.s[1],w14
2499
	mov	v3.s[2],w13
2500
	mov	v3.s[3],w12
2501
#ifndef __AARCH64EB__
2502
	rev32	v3.16b,v3.16b
2503
#endif
2504
	ld1	{v4.4s},[x0]
2505
	eor	v4.16b,v4.16b,v3.16b
2506
	st1	{v4.4s},[x1]
2507
	b	100f
2508
1:	//	last 2 blocks processing
2509
	dup	v4.4s,w12
2510
	dup	v5.4s,w13
2511
	dup	v6.4s,w14
2512
	mov	v7.s[0],w5
2513
	add	w5,w5,#1
2514
	mov	v7.s[1],w5
2515
	subs	w2,w2,#1
2516
	b.ne	1f
2517
	bl	_vpsm4_ex_enc_4blks
2518
	ld4	{v12.s,v13.s,v14.s,v15.s}[0],[x0],#16
2519
	ld4	{v12.s,v13.s,v14.s,v15.s}[1],[x0],#16
2520
	eor	v0.16b,v0.16b,v12.16b
2521
	eor	v1.16b,v1.16b,v13.16b
2522
	eor	v2.16b,v2.16b,v14.16b
2523
	eor	v3.16b,v3.16b,v15.16b
2524
	st4	{v0.s,v1.s,v2.s,v3.s}[0],[x1],#16
2525
	st4	{v0.s,v1.s,v2.s,v3.s}[1],[x1],#16
2526
	b	100f
2527
1:	//	last 3 blocks processing
2528
	add	w5,w5,#1
2529
	mov	v7.s[2],w5
2530
	bl	_vpsm4_ex_enc_4blks
2531
	ld4	{v12.s,v13.s,v14.s,v15.s}[0],[x0],#16
2532
	ld4	{v12.s,v13.s,v14.s,v15.s}[1],[x0],#16
2533
	ld4	{v12.s,v13.s,v14.s,v15.s}[2],[x0],#16
2534
	eor	v0.16b,v0.16b,v12.16b
2535
	eor	v1.16b,v1.16b,v13.16b
2536
	eor	v2.16b,v2.16b,v14.16b
2537
	eor	v3.16b,v3.16b,v15.16b
2538
	st4	{v0.s,v1.s,v2.s,v3.s}[0],[x1],#16
2539
	st4	{v0.s,v1.s,v2.s,v3.s}[1],[x1],#16
2540
	st4	{v0.s,v1.s,v2.s,v3.s}[2],[x1],#16
2541
100:
2542
	ldp	d10,d11,[sp,#16]
2543
	ldp	d12,d13,[sp,#32]
2544
	ldp	d14,d15,[sp,#48]
2545
	ldp	x29,x30,[sp,#64]
2546
	ldp	d8,d9,[sp],#80
2547
	AARCH64_VALIDATE_LINK_REGISTER
2548
	ret
2549
.size	vpsm4_ex_ctr32_encrypt_blocks,.-vpsm4_ex_ctr32_encrypt_blocks
2550
.globl	vpsm4_ex_xts_encrypt_gb
2551
.type	vpsm4_ex_xts_encrypt_gb,%function
2552
.align	5
2553
vpsm4_ex_xts_encrypt_gb:
2554
	AARCH64_SIGN_LINK_REGISTER
2555
	stp	x15, x16, [sp, #-0x10]!
2556
	stp	x17, x18, [sp, #-0x10]!
2557
	stp	x19, x20, [sp, #-0x10]!
2558
	stp	x21, x22, [sp, #-0x10]!
2559
	stp	x23, x24, [sp, #-0x10]!
2560
	stp	x25, x26, [sp, #-0x10]!
2561
	stp	x27, x28, [sp, #-0x10]!
2562
	stp	x29, x30, [sp, #-0x10]!
2563
	stp	d8, d9, [sp, #-0x10]!
2564
	stp	d10, d11, [sp, #-0x10]!
2565
	stp	d12, d13, [sp, #-0x10]!
2566
	stp	d14, d15, [sp, #-0x10]!
2567
	mov	x26,x3
2568
	mov	x27,x4
2569
	mov	w28,w6
2570
	ld1	{v16.4s}, [x5]
2571
	mov	x3,x27
2572
	adrp	x9, .Lsbox_magic
2573
	ldr	q26, [x9, #:lo12:.Lsbox_magic]
2574
	ldr	q27, [x9, #:lo12:.Lsbox_magic+16]
2575
	ldr	q28, [x9, #:lo12:.Lsbox_magic+32]
2576
	ldr	q29, [x9, #:lo12:.Lsbox_magic+48]
2577
	ldr	q30, [x9, #:lo12:.Lsbox_magic+64]
2578
	ldr	q31, [x9, #:lo12:.Lsbox_magic+80]
2579
#ifndef __AARCH64EB__
2580
	rev32	v16.16b,v16.16b
2581
#endif
2582
	mov	x10,x3
2583
	mov	w11,#8
2584
	mov	w12,v16.s[0]
2585
	mov	w13,v16.s[1]
2586
	mov	w14,v16.s[2]
2587
	mov	w15,v16.s[3]
2588
10:
2589
	ldp	w7,w8,[x10],8
2590
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
2591
	eor	w6,w14,w15
2592
	eor	w9,w7,w13
2593
	eor	w6,w6,w9
2594
	mov	v3.s[0],w6
2595
	// optimize sbox using AESE instruction
2596
	tbl	v0.16b, {v3.16b}, v26.16b
2597
	ushr	v2.16b, v0.16b, 4
2598
	and	v0.16b, v0.16b, v31.16b
2599
	tbl	v0.16b, {v28.16b}, v0.16b
2600
	tbl	v2.16b, {v27.16b}, v2.16b
2601
	eor	v0.16b, v0.16b, v2.16b
2602
	eor	v1.16b, v1.16b, v1.16b
2603
	aese	v0.16b,v1.16b
2604
	ushr	v2.16b, v0.16b, 4
2605
	and	v0.16b, v0.16b, v31.16b
2606
	tbl	v0.16b, {v30.16b}, v0.16b
2607
	tbl	v2.16b, {v29.16b}, v2.16b
2608
	eor	v0.16b, v0.16b, v2.16b
2609

2610
	mov	w7,v0.s[0]
2611
	eor	w6,w7,w7,ror #32-2
2612
	eor	w6,w6,w7,ror #32-10
2613
	eor	w6,w6,w7,ror #32-18
2614
	eor	w6,w6,w7,ror #32-24
2615
	eor	w12,w12,w6
2616
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
2617
	eor	w6,w14,w15
2618
	eor	w9,w12,w8
2619
	eor	w6,w6,w9
2620
	mov	v3.s[0],w6
2621
	// optimize sbox using AESE instruction
2622
	tbl	v0.16b, {v3.16b}, v26.16b
2623
	ushr	v2.16b, v0.16b, 4
2624
	and	v0.16b, v0.16b, v31.16b
2625
	tbl	v0.16b, {v28.16b}, v0.16b
2626
	tbl	v2.16b, {v27.16b}, v2.16b
2627
	eor	v0.16b, v0.16b, v2.16b
2628
	eor	v1.16b, v1.16b, v1.16b
2629
	aese	v0.16b,v1.16b
2630
	ushr	v2.16b, v0.16b, 4
2631
	and	v0.16b, v0.16b, v31.16b
2632
	tbl	v0.16b, {v30.16b}, v0.16b
2633
	tbl	v2.16b, {v29.16b}, v2.16b
2634
	eor	v0.16b, v0.16b, v2.16b
2635

2636
	mov	w7,v0.s[0]
2637
	eor	w6,w7,w7,ror #32-2
2638
	eor	w6,w6,w7,ror #32-10
2639
	eor	w6,w6,w7,ror #32-18
2640
	eor	w6,w6,w7,ror #32-24
2641
	ldp	w7,w8,[x10],8
2642
	eor	w13,w13,w6
2643
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
2644
	eor	w6,w12,w13
2645
	eor	w9,w7,w15
2646
	eor	w6,w6,w9
2647
	mov	v3.s[0],w6
2648
	// optimize sbox using AESE instruction
2649
	tbl	v0.16b, {v3.16b}, v26.16b
2650
	ushr	v2.16b, v0.16b, 4
2651
	and	v0.16b, v0.16b, v31.16b
2652
	tbl	v0.16b, {v28.16b}, v0.16b
2653
	tbl	v2.16b, {v27.16b}, v2.16b
2654
	eor	v0.16b, v0.16b, v2.16b
2655
	eor	v1.16b, v1.16b, v1.16b
2656
	aese	v0.16b,v1.16b
2657
	ushr	v2.16b, v0.16b, 4
2658
	and	v0.16b, v0.16b, v31.16b
2659
	tbl	v0.16b, {v30.16b}, v0.16b
2660
	tbl	v2.16b, {v29.16b}, v2.16b
2661
	eor	v0.16b, v0.16b, v2.16b
2662

2663
	mov	w7,v0.s[0]
2664
	eor	w6,w7,w7,ror #32-2
2665
	eor	w6,w6,w7,ror #32-10
2666
	eor	w6,w6,w7,ror #32-18
2667
	eor	w6,w6,w7,ror #32-24
2668
	eor	w14,w14,w6
2669
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
2670
	eor	w6,w12,w13
2671
	eor	w9,w14,w8
2672
	eor	w6,w6,w9
2673
	mov	v3.s[0],w6
2674
	// optimize sbox using AESE instruction
2675
	tbl	v0.16b, {v3.16b}, v26.16b
2676
	ushr	v2.16b, v0.16b, 4
2677
	and	v0.16b, v0.16b, v31.16b
2678
	tbl	v0.16b, {v28.16b}, v0.16b
2679
	tbl	v2.16b, {v27.16b}, v2.16b
2680
	eor	v0.16b, v0.16b, v2.16b
2681
	eor	v1.16b, v1.16b, v1.16b
2682
	aese	v0.16b,v1.16b
2683
	ushr	v2.16b, v0.16b, 4
2684
	and	v0.16b, v0.16b, v31.16b
2685
	tbl	v0.16b, {v30.16b}, v0.16b
2686
	tbl	v2.16b, {v29.16b}, v2.16b
2687
	eor	v0.16b, v0.16b, v2.16b
2688

2689
	mov	w7,v0.s[0]
2690
	eor	w6,w7,w7,ror #32-2
2691
	eor	w6,w6,w7,ror #32-10
2692
	eor	w6,w6,w7,ror #32-18
2693
	eor	w6,w6,w7,ror #32-24
2694
	eor	w15,w15,w6
2695
	subs	w11,w11,#1
2696
	b.ne	10b
2697
	mov	v16.s[0],w15
2698
	mov	v16.s[1],w14
2699
	mov	v16.s[2],w13
2700
	mov	v16.s[3],w12
2701
#ifndef __AARCH64EB__
2702
	rev32	v16.16b,v16.16b
2703
#endif
2704
	mov	x3,x26
2705
	and	x29,x2,#0x0F
2706
	// convert length into blocks
2707
	lsr	x2,x2,4
2708
	cmp	x2,#1
2709
	b.lt	.return_gb
2710

2711
	cmp	x29,0
2712
	// If the encryption/decryption Length is N times of 16,
2713
	// the all blocks are encrypted/decrypted in .xts_encrypt_blocks_gb
2714
	b.eq	.xts_encrypt_blocks_gb
2715

2716
	// If the encryption/decryption length is not N times of 16,
2717
	// the last two blocks are encrypted/decrypted in .last_2blks_tweak_gb or .only_2blks_tweak_gb
2718
	// the other blocks are encrypted/decrypted in .xts_encrypt_blocks_gb
2719
	subs	x2,x2,#1
2720
	b.eq	.only_2blks_tweak_gb
2721
.xts_encrypt_blocks_gb:
2722
	rbit	v16.16b,v16.16b
2723
#ifdef __AARCH64EB__
2724
	rev32	v16.16b,v16.16b
2725
#endif
2726
	mov	x12,v16.d[0]
2727
	mov	x13,v16.d[1]
2728
	mov	w7,0x87
2729
	extr	x9,x13,x13,#32
2730
	extr	x15,x13,x12,#63
2731
	and	w8,w7,w9,asr#31
2732
	eor	x14,x8,x12,lsl#1
2733
	mov	w7,0x87
2734
	extr	x9,x15,x15,#32
2735
	extr	x17,x15,x14,#63
2736
	and	w8,w7,w9,asr#31
2737
	eor	x16,x8,x14,lsl#1
2738
	mov	w7,0x87
2739
	extr	x9,x17,x17,#32
2740
	extr	x19,x17,x16,#63
2741
	and	w8,w7,w9,asr#31
2742
	eor	x18,x8,x16,lsl#1
2743
	mov	w7,0x87
2744
	extr	x9,x19,x19,#32
2745
	extr	x21,x19,x18,#63
2746
	and	w8,w7,w9,asr#31
2747
	eor	x20,x8,x18,lsl#1
2748
	mov	w7,0x87
2749
	extr	x9,x21,x21,#32
2750
	extr	x23,x21,x20,#63
2751
	and	w8,w7,w9,asr#31
2752
	eor	x22,x8,x20,lsl#1
2753
	mov	w7,0x87
2754
	extr	x9,x23,x23,#32
2755
	extr	x25,x23,x22,#63
2756
	and	w8,w7,w9,asr#31
2757
	eor	x24,x8,x22,lsl#1
2758
	mov	w7,0x87
2759
	extr	x9,x25,x25,#32
2760
	extr	x27,x25,x24,#63
2761
	and	w8,w7,w9,asr#31
2762
	eor	x26,x8,x24,lsl#1
2763
.Lxts_8_blocks_process_gb:
2764
	cmp	x2,#8
2765
	mov	v16.d[0],x12
2766
	mov	v16.d[1],x13
2767
#ifdef __AARCH64EB__
2768
	rev32	v16.16b,v16.16b
2769
#endif
2770
	mov	w7,0x87
2771
	extr	x9,x27,x27,#32
2772
	extr	x13,x27,x26,#63
2773
	and	w8,w7,w9,asr#31
2774
	eor	x12,x8,x26,lsl#1
2775
	mov	v17.d[0],x14
2776
	mov	v17.d[1],x15
2777
#ifdef __AARCH64EB__
2778
	rev32	v17.16b,v17.16b
2779
#endif
2780
	mov	w7,0x87
2781
	extr	x9,x13,x13,#32
2782
	extr	x15,x13,x12,#63
2783
	and	w8,w7,w9,asr#31
2784
	eor	x14,x8,x12,lsl#1
2785
	mov	v18.d[0],x16
2786
	mov	v18.d[1],x17
2787
#ifdef __AARCH64EB__
2788
	rev32	v18.16b,v18.16b
2789
#endif
2790
	mov	w7,0x87
2791
	extr	x9,x15,x15,#32
2792
	extr	x17,x15,x14,#63
2793
	and	w8,w7,w9,asr#31
2794
	eor	x16,x8,x14,lsl#1
2795
	mov	v19.d[0],x18
2796
	mov	v19.d[1],x19
2797
#ifdef __AARCH64EB__
2798
	rev32	v19.16b,v19.16b
2799
#endif
2800
	mov	w7,0x87
2801
	extr	x9,x17,x17,#32
2802
	extr	x19,x17,x16,#63
2803
	and	w8,w7,w9,asr#31
2804
	eor	x18,x8,x16,lsl#1
2805
	mov	v20.d[0],x20
2806
	mov	v20.d[1],x21
2807
#ifdef __AARCH64EB__
2808
	rev32	v20.16b,v20.16b
2809
#endif
2810
	mov	w7,0x87
2811
	extr	x9,x19,x19,#32
2812
	extr	x21,x19,x18,#63
2813
	and	w8,w7,w9,asr#31
2814
	eor	x20,x8,x18,lsl#1
2815
	mov	v21.d[0],x22
2816
	mov	v21.d[1],x23
2817
#ifdef __AARCH64EB__
2818
	rev32	v21.16b,v21.16b
2819
#endif
2820
	mov	w7,0x87
2821
	extr	x9,x21,x21,#32
2822
	extr	x23,x21,x20,#63
2823
	and	w8,w7,w9,asr#31
2824
	eor	x22,x8,x20,lsl#1
2825
	mov	v22.d[0],x24
2826
	mov	v22.d[1],x25
2827
#ifdef __AARCH64EB__
2828
	rev32	v22.16b,v22.16b
2829
#endif
2830
	mov	w7,0x87
2831
	extr	x9,x23,x23,#32
2832
	extr	x25,x23,x22,#63
2833
	and	w8,w7,w9,asr#31
2834
	eor	x24,x8,x22,lsl#1
2835
	mov	v23.d[0],x26
2836
	mov	v23.d[1],x27
2837
#ifdef __AARCH64EB__
2838
	rev32	v23.16b,v23.16b
2839
#endif
2840
	mov	w7,0x87
2841
	extr	x9,x25,x25,#32
2842
	extr	x27,x25,x24,#63
2843
	and	w8,w7,w9,asr#31
2844
	eor	x26,x8,x24,lsl#1
2845
	b.lt	.Lxts_4_blocks_process_gb
2846
	ld1	{v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64
2847
	rbit	v16.16b,v16.16b
2848
	rbit	v17.16b,v17.16b
2849
	rbit	v18.16b,v18.16b
2850
	rbit	v19.16b,v19.16b
2851
	eor	v4.16b, v4.16b, v16.16b
2852
	eor	v5.16b, v5.16b, v17.16b
2853
	eor	v6.16b, v6.16b, v18.16b
2854
	eor	v7.16b, v7.16b, v19.16b
2855
	ld1	{v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64
2856
	rbit	v20.16b,v20.16b
2857
	rbit	v21.16b,v21.16b
2858
	rbit	v22.16b,v22.16b
2859
	rbit	v23.16b,v23.16b
2860
	eor	v8.16b, v8.16b, v20.16b
2861
	eor	v9.16b, v9.16b, v21.16b
2862
	eor	v10.16b, v10.16b, v22.16b
2863
	eor	v11.16b, v11.16b, v23.16b
2864
#ifndef __AARCH64EB__
2865
	rev32	v4.16b,v4.16b
2866
#endif
2867
#ifndef __AARCH64EB__
2868
	rev32	v5.16b,v5.16b
2869
#endif
2870
#ifndef __AARCH64EB__
2871
	rev32	v6.16b,v6.16b
2872
#endif
2873
#ifndef __AARCH64EB__
2874
	rev32	v7.16b,v7.16b
2875
#endif
2876
#ifndef __AARCH64EB__
2877
	rev32	v8.16b,v8.16b
2878
#endif
2879
#ifndef __AARCH64EB__
2880
	rev32	v9.16b,v9.16b
2881
#endif
2882
#ifndef __AARCH64EB__
2883
	rev32	v10.16b,v10.16b
2884
#endif
2885
#ifndef __AARCH64EB__
2886
	rev32	v11.16b,v11.16b
2887
#endif
2888
	zip1	v0.4s,v4.4s,v5.4s
2889
	zip2	v1.4s,v4.4s,v5.4s
2890
	zip1	v2.4s,v6.4s,v7.4s
2891
	zip2	v3.4s,v6.4s,v7.4s
2892
	zip1	v4.2d,v0.2d,v2.2d
2893
	zip2	v5.2d,v0.2d,v2.2d
2894
	zip1	v6.2d,v1.2d,v3.2d
2895
	zip2	v7.2d,v1.2d,v3.2d
2896
	zip1	v0.4s,v8.4s,v9.4s
2897
	zip2	v1.4s,v8.4s,v9.4s
2898
	zip1	v2.4s,v10.4s,v11.4s
2899
	zip2	v3.4s,v10.4s,v11.4s
2900
	zip1	v8.2d,v0.2d,v2.2d
2901
	zip2	v9.2d,v0.2d,v2.2d
2902
	zip1	v10.2d,v1.2d,v3.2d
2903
	zip2	v11.2d,v1.2d,v3.2d
2904
	bl	_vpsm4_ex_enc_8blks
2905
	zip1	v8.4s,v0.4s,v1.4s
2906
	zip2	v9.4s,v0.4s,v1.4s
2907
	zip1	v10.4s,v2.4s,v3.4s
2908
	zip2	v11.4s,v2.4s,v3.4s
2909
	zip1	v0.2d,v8.2d,v10.2d
2910
	zip2	v1.2d,v8.2d,v10.2d
2911
	zip1	v2.2d,v9.2d,v11.2d
2912
	zip2	v3.2d,v9.2d,v11.2d
2913
	zip1	v8.4s,v4.4s,v5.4s
2914
	zip2	v9.4s,v4.4s,v5.4s
2915
	zip1	v10.4s,v6.4s,v7.4s
2916
	zip2	v11.4s,v6.4s,v7.4s
2917
	zip1	v4.2d,v8.2d,v10.2d
2918
	zip2	v5.2d,v8.2d,v10.2d
2919
	zip1	v6.2d,v9.2d,v11.2d
2920
	zip2	v7.2d,v9.2d,v11.2d
2921
	eor	v0.16b, v0.16b, v16.16b
2922
	eor	v1.16b, v1.16b, v17.16b
2923
	eor	v2.16b, v2.16b, v18.16b
2924
	eor	v3.16b, v3.16b, v19.16b
2925
	eor	v4.16b, v4.16b, v20.16b
2926
	eor	v5.16b, v5.16b, v21.16b
2927
	eor	v6.16b, v6.16b, v22.16b
2928
	eor	v7.16b, v7.16b, v23.16b
2929

2930
	// save the last tweak
2931
	mov	v25.16b,v23.16b
2932
	st1	{v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64
2933
	st1	{v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64
2934
	subs	x2,x2,#8
2935
	b.gt	.Lxts_8_blocks_process_gb
2936
	b	100f
2937
.Lxts_4_blocks_process_gb:
2938
	cmp	x2,#4
2939
	b.lt	1f
2940
	ld1	{v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64
2941
	rbit	v16.16b,v16.16b
2942
	rbit	v17.16b,v17.16b
2943
	rbit	v18.16b,v18.16b
2944
	rbit	v19.16b,v19.16b
2945
	eor	v4.16b, v4.16b, v16.16b
2946
	eor	v5.16b, v5.16b, v17.16b
2947
	eor	v6.16b, v6.16b, v18.16b
2948
	eor	v7.16b, v7.16b, v19.16b
2949
#ifndef __AARCH64EB__
2950
	rev32	v4.16b,v4.16b
2951
#endif
2952
#ifndef __AARCH64EB__
2953
	rev32	v5.16b,v5.16b
2954
#endif
2955
#ifndef __AARCH64EB__
2956
	rev32	v6.16b,v6.16b
2957
#endif
2958
#ifndef __AARCH64EB__
2959
	rev32	v7.16b,v7.16b
2960
#endif
2961
	zip1	v0.4s,v4.4s,v5.4s
2962
	zip2	v1.4s,v4.4s,v5.4s
2963
	zip1	v2.4s,v6.4s,v7.4s
2964
	zip2	v3.4s,v6.4s,v7.4s
2965
	zip1	v4.2d,v0.2d,v2.2d
2966
	zip2	v5.2d,v0.2d,v2.2d
2967
	zip1	v6.2d,v1.2d,v3.2d
2968
	zip2	v7.2d,v1.2d,v3.2d
2969
	bl	_vpsm4_ex_enc_4blks
2970
	zip1	v4.4s,v0.4s,v1.4s
2971
	zip2	v5.4s,v0.4s,v1.4s
2972
	zip1	v6.4s,v2.4s,v3.4s
2973
	zip2	v7.4s,v2.4s,v3.4s
2974
	zip1	v0.2d,v4.2d,v6.2d
2975
	zip2	v1.2d,v4.2d,v6.2d
2976
	zip1	v2.2d,v5.2d,v7.2d
2977
	zip2	v3.2d,v5.2d,v7.2d
2978
	eor	v0.16b, v0.16b, v16.16b
2979
	eor	v1.16b, v1.16b, v17.16b
2980
	eor	v2.16b, v2.16b, v18.16b
2981
	eor	v3.16b, v3.16b, v19.16b
2982
	st1	{v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64
2983
	sub	x2,x2,#4
2984
	mov	v16.16b,v20.16b
2985
	mov	v17.16b,v21.16b
2986
	mov	v18.16b,v22.16b
2987
	// save the last tweak
2988
	mov	v25.16b,v19.16b
2989
1:
2990
	// process last block
2991
	cmp	x2,#1
2992
	b.lt	100f
2993
	b.gt	1f
2994
	ld1	{v4.4s},[x0],#16
2995
	rbit	v16.16b,v16.16b
2996
	eor	v4.16b, v4.16b, v16.16b
2997
#ifndef __AARCH64EB__
2998
	rev32	v4.16b,v4.16b
2999
#endif
3000
	mov	x10,x3
3001
	mov	w11,#8
3002
	mov	w12,v4.s[0]
3003
	mov	w13,v4.s[1]
3004
	mov	w14,v4.s[2]
3005
	mov	w15,v4.s[3]
3006
10:
3007
	ldp	w7,w8,[x10],8
3008
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
3009
	eor	w6,w14,w15
3010
	eor	w9,w7,w13
3011
	eor	w6,w6,w9
3012
	mov	v3.s[0],w6
3013
	// optimize sbox using AESE instruction
3014
	tbl	v0.16b, {v3.16b}, v26.16b
3015
	ushr	v2.16b, v0.16b, 4
3016
	and	v0.16b, v0.16b, v31.16b
3017
	tbl	v0.16b, {v28.16b}, v0.16b
3018
	tbl	v2.16b, {v27.16b}, v2.16b
3019
	eor	v0.16b, v0.16b, v2.16b
3020
	eor	v1.16b, v1.16b, v1.16b
3021
	aese	v0.16b,v1.16b
3022
	ushr	v2.16b, v0.16b, 4
3023
	and	v0.16b, v0.16b, v31.16b
3024
	tbl	v0.16b, {v30.16b}, v0.16b
3025
	tbl	v2.16b, {v29.16b}, v2.16b
3026
	eor	v0.16b, v0.16b, v2.16b
3027

3028
	mov	w7,v0.s[0]
3029
	eor	w6,w7,w7,ror #32-2
3030
	eor	w6,w6,w7,ror #32-10
3031
	eor	w6,w6,w7,ror #32-18
3032
	eor	w6,w6,w7,ror #32-24
3033
	eor	w12,w12,w6
3034
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
3035
	eor	w6,w14,w15
3036
	eor	w9,w12,w8
3037
	eor	w6,w6,w9
3038
	mov	v3.s[0],w6
3039
	// optimize sbox using AESE instruction
3040
	tbl	v0.16b, {v3.16b}, v26.16b
3041
	ushr	v2.16b, v0.16b, 4
3042
	and	v0.16b, v0.16b, v31.16b
3043
	tbl	v0.16b, {v28.16b}, v0.16b
3044
	tbl	v2.16b, {v27.16b}, v2.16b
3045
	eor	v0.16b, v0.16b, v2.16b
3046
	eor	v1.16b, v1.16b, v1.16b
3047
	aese	v0.16b,v1.16b
3048
	ushr	v2.16b, v0.16b, 4
3049
	and	v0.16b, v0.16b, v31.16b
3050
	tbl	v0.16b, {v30.16b}, v0.16b
3051
	tbl	v2.16b, {v29.16b}, v2.16b
3052
	eor	v0.16b, v0.16b, v2.16b
3053

3054
	mov	w7,v0.s[0]
3055
	eor	w6,w7,w7,ror #32-2
3056
	eor	w6,w6,w7,ror #32-10
3057
	eor	w6,w6,w7,ror #32-18
3058
	eor	w6,w6,w7,ror #32-24
3059
	ldp	w7,w8,[x10],8
3060
	eor	w13,w13,w6
3061
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
3062
	eor	w6,w12,w13
3063
	eor	w9,w7,w15
3064
	eor	w6,w6,w9
3065
	mov	v3.s[0],w6
3066
	// optimize sbox using AESE instruction
3067
	tbl	v0.16b, {v3.16b}, v26.16b
3068
	ushr	v2.16b, v0.16b, 4
3069
	and	v0.16b, v0.16b, v31.16b
3070
	tbl	v0.16b, {v28.16b}, v0.16b
3071
	tbl	v2.16b, {v27.16b}, v2.16b
3072
	eor	v0.16b, v0.16b, v2.16b
3073
	eor	v1.16b, v1.16b, v1.16b
3074
	aese	v0.16b,v1.16b
3075
	ushr	v2.16b, v0.16b, 4
3076
	and	v0.16b, v0.16b, v31.16b
3077
	tbl	v0.16b, {v30.16b}, v0.16b
3078
	tbl	v2.16b, {v29.16b}, v2.16b
3079
	eor	v0.16b, v0.16b, v2.16b
3080

3081
	mov	w7,v0.s[0]
3082
	eor	w6,w7,w7,ror #32-2
3083
	eor	w6,w6,w7,ror #32-10
3084
	eor	w6,w6,w7,ror #32-18
3085
	eor	w6,w6,w7,ror #32-24
3086
	eor	w14,w14,w6
3087
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
3088
	eor	w6,w12,w13
3089
	eor	w9,w14,w8
3090
	eor	w6,w6,w9
3091
	mov	v3.s[0],w6
3092
	// optimize sbox using AESE instruction
3093
	tbl	v0.16b, {v3.16b}, v26.16b
3094
	ushr	v2.16b, v0.16b, 4
3095
	and	v0.16b, v0.16b, v31.16b
3096
	tbl	v0.16b, {v28.16b}, v0.16b
3097
	tbl	v2.16b, {v27.16b}, v2.16b
3098
	eor	v0.16b, v0.16b, v2.16b
3099
	eor	v1.16b, v1.16b, v1.16b
3100
	aese	v0.16b,v1.16b
3101
	ushr	v2.16b, v0.16b, 4
3102
	and	v0.16b, v0.16b, v31.16b
3103
	tbl	v0.16b, {v30.16b}, v0.16b
3104
	tbl	v2.16b, {v29.16b}, v2.16b
3105
	eor	v0.16b, v0.16b, v2.16b
3106

3107
	mov	w7,v0.s[0]
3108
	eor	w6,w7,w7,ror #32-2
3109
	eor	w6,w6,w7,ror #32-10
3110
	eor	w6,w6,w7,ror #32-18
3111
	eor	w6,w6,w7,ror #32-24
3112
	eor	w15,w15,w6
3113
	subs	w11,w11,#1
3114
	b.ne	10b
3115
	mov	v4.s[0],w15
3116
	mov	v4.s[1],w14
3117
	mov	v4.s[2],w13
3118
	mov	v4.s[3],w12
3119
#ifndef __AARCH64EB__
3120
	rev32	v4.16b,v4.16b
3121
#endif
3122
	eor	v4.16b, v4.16b, v16.16b
3123
	st1	{v4.4s},[x1],#16
3124
	// save the last tweak
3125
	mov	v25.16b,v16.16b
3126
	b	100f
3127
1:	//	process last 2 blocks
3128
	cmp	x2,#2
3129
	b.gt	1f
3130
	ld1	{v4.4s,v5.4s},[x0],#32
3131
	rbit	v16.16b,v16.16b
3132
	rbit	v17.16b,v17.16b
3133
	eor	v4.16b, v4.16b, v16.16b
3134
	eor	v5.16b, v5.16b, v17.16b
3135
#ifndef __AARCH64EB__
3136
	rev32	v4.16b,v4.16b
3137
#endif
3138
#ifndef __AARCH64EB__
3139
	rev32	v5.16b,v5.16b
3140
#endif
3141
	zip1	v0.4s,v4.4s,v5.4s
3142
	zip2	v1.4s,v4.4s,v5.4s
3143
	zip1	v2.4s,v6.4s,v7.4s
3144
	zip2	v3.4s,v6.4s,v7.4s
3145
	zip1	v4.2d,v0.2d,v2.2d
3146
	zip2	v5.2d,v0.2d,v2.2d
3147
	zip1	v6.2d,v1.2d,v3.2d
3148
	zip2	v7.2d,v1.2d,v3.2d
3149
	bl	_vpsm4_ex_enc_4blks
3150
	zip1	v4.4s,v0.4s,v1.4s
3151
	zip2	v5.4s,v0.4s,v1.4s
3152
	zip1	v6.4s,v2.4s,v3.4s
3153
	zip2	v7.4s,v2.4s,v3.4s
3154
	zip1	v0.2d,v4.2d,v6.2d
3155
	zip2	v1.2d,v4.2d,v6.2d
3156
	zip1	v2.2d,v5.2d,v7.2d
3157
	zip2	v3.2d,v5.2d,v7.2d
3158
	eor	v0.16b, v0.16b, v16.16b
3159
	eor	v1.16b, v1.16b, v17.16b
3160
	st1	{v0.4s,v1.4s},[x1],#32
3161
	// save the last tweak
3162
	mov	v25.16b,v17.16b
3163
	b	100f
3164
1:	//	process last 3 blocks
3165
	ld1	{v4.4s,v5.4s,v6.4s},[x0],#48
3166
	rbit	v16.16b,v16.16b
3167
	rbit	v17.16b,v17.16b
3168
	rbit	v18.16b,v18.16b
3169
	eor	v4.16b, v4.16b, v16.16b
3170
	eor	v5.16b, v5.16b, v17.16b
3171
	eor	v6.16b, v6.16b, v18.16b
3172
#ifndef __AARCH64EB__
3173
	rev32	v4.16b,v4.16b
3174
#endif
3175
#ifndef __AARCH64EB__
3176
	rev32	v5.16b,v5.16b
3177
#endif
3178
#ifndef __AARCH64EB__
3179
	rev32	v6.16b,v6.16b
3180
#endif
3181
	zip1	v0.4s,v4.4s,v5.4s
3182
	zip2	v1.4s,v4.4s,v5.4s
3183
	zip1	v2.4s,v6.4s,v7.4s
3184
	zip2	v3.4s,v6.4s,v7.4s
3185
	zip1	v4.2d,v0.2d,v2.2d
3186
	zip2	v5.2d,v0.2d,v2.2d
3187
	zip1	v6.2d,v1.2d,v3.2d
3188
	zip2	v7.2d,v1.2d,v3.2d
3189
	bl	_vpsm4_ex_enc_4blks
3190
	zip1	v4.4s,v0.4s,v1.4s
3191
	zip2	v5.4s,v0.4s,v1.4s
3192
	zip1	v6.4s,v2.4s,v3.4s
3193
	zip2	v7.4s,v2.4s,v3.4s
3194
	zip1	v0.2d,v4.2d,v6.2d
3195
	zip2	v1.2d,v4.2d,v6.2d
3196
	zip1	v2.2d,v5.2d,v7.2d
3197
	zip2	v3.2d,v5.2d,v7.2d
3198
	eor	v0.16b, v0.16b, v16.16b
3199
	eor	v1.16b, v1.16b, v17.16b
3200
	eor	v2.16b, v2.16b, v18.16b
3201
	st1	{v0.4s,v1.4s,v2.4s},[x1],#48
3202
	// save the last tweak
3203
	mov	v25.16b,v18.16b
3204
100:
3205
	cmp	x29,0
3206
	b.eq	.return_gb
3207

3208
// This branch calculates the last two tweaks, 
3209
// while the encryption/decryption length is larger than 32
3210
.last_2blks_tweak_gb:
3211
#ifdef __AARCH64EB__
3212
	rev32	v25.16b,v25.16b
3213
#endif
3214
	rbit	v2.16b,v25.16b
3215
	adrp	x9, .Lxts_magic
3216
	ldr	q0, [x9, #:lo12:.Lxts_magic]
3217
	shl	v17.16b, v2.16b, #1
3218
	ext	v1.16b, v2.16b, v2.16b,#15
3219
	ushr	v1.16b, v1.16b, #7
3220
	mul	v1.16b, v1.16b, v0.16b
3221
	eor	v17.16b, v17.16b, v1.16b
3222
	rbit	v17.16b,v17.16b
3223
	rbit	v2.16b,v17.16b
3224
	adrp	x9, .Lxts_magic
3225
	ldr	q0, [x9, #:lo12:.Lxts_magic]
3226
	shl	v18.16b, v2.16b, #1
3227
	ext	v1.16b, v2.16b, v2.16b,#15
3228
	ushr	v1.16b, v1.16b, #7
3229
	mul	v1.16b, v1.16b, v0.16b
3230
	eor	v18.16b, v18.16b, v1.16b
3231
	rbit	v18.16b,v18.16b
3232
	b	.check_dec_gb
3233

3234

3235
// This branch calculates the last two tweaks, 
3236
// while the encryption/decryption length is equal to 32, who only need two tweaks
3237
.only_2blks_tweak_gb:
3238
	mov	v17.16b,v16.16b
3239
#ifdef __AARCH64EB__
3240
	rev32	v17.16b,v17.16b
3241
#endif
3242
	rbit	v2.16b,v17.16b
3243
	adrp	x9, .Lxts_magic
3244
	ldr	q0, [x9, #:lo12:.Lxts_magic]
3245
	shl	v18.16b, v2.16b, #1
3246
	ext	v1.16b, v2.16b, v2.16b,#15
3247
	ushr	v1.16b, v1.16b, #7
3248
	mul	v1.16b, v1.16b, v0.16b
3249
	eor	v18.16b, v18.16b, v1.16b
3250
	rbit	v18.16b,v18.16b
3251
	b	.check_dec_gb
3252

3253

3254
// Determine whether encryption or decryption is required.
3255
// The last two tweaks need to be swapped for decryption.
3256
.check_dec_gb:
3257
	// encryption:1 decryption:0
3258
	cmp	w28,1
3259
	b.eq	.process_last_2blks_gb
3260
	mov	v0.16B,v17.16b
3261
	mov	v17.16B,v18.16b
3262
	mov	v18.16B,v0.16b
3263

3264
.process_last_2blks_gb:
3265
#ifdef __AARCH64EB__
3266
	rev32	v17.16b,v17.16b
3267
#endif
3268
#ifdef __AARCH64EB__
3269
	rev32	v18.16b,v18.16b
3270
#endif
3271
	ld1	{v4.4s},[x0],#16
3272
	eor	v4.16b, v4.16b, v17.16b
3273
#ifndef __AARCH64EB__
3274
	rev32	v4.16b,v4.16b
3275
#endif
3276
	mov	x10,x3
3277
	mov	w11,#8
3278
	mov	w12,v4.s[0]
3279
	mov	w13,v4.s[1]
3280
	mov	w14,v4.s[2]
3281
	mov	w15,v4.s[3]
3282
10:
3283
	ldp	w7,w8,[x10],8
3284
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
3285
	eor	w6,w14,w15
3286
	eor	w9,w7,w13
3287
	eor	w6,w6,w9
3288
	mov	v3.s[0],w6
3289
	// optimize sbox using AESE instruction
3290
	tbl	v0.16b, {v3.16b}, v26.16b
3291
	ushr	v2.16b, v0.16b, 4
3292
	and	v0.16b, v0.16b, v31.16b
3293
	tbl	v0.16b, {v28.16b}, v0.16b
3294
	tbl	v2.16b, {v27.16b}, v2.16b
3295
	eor	v0.16b, v0.16b, v2.16b
3296
	eor	v1.16b, v1.16b, v1.16b
3297
	aese	v0.16b,v1.16b
3298
	ushr	v2.16b, v0.16b, 4
3299
	and	v0.16b, v0.16b, v31.16b
3300
	tbl	v0.16b, {v30.16b}, v0.16b
3301
	tbl	v2.16b, {v29.16b}, v2.16b
3302
	eor	v0.16b, v0.16b, v2.16b
3303

3304
	mov	w7,v0.s[0]
3305
	eor	w6,w7,w7,ror #32-2
3306
	eor	w6,w6,w7,ror #32-10
3307
	eor	w6,w6,w7,ror #32-18
3308
	eor	w6,w6,w7,ror #32-24
3309
	eor	w12,w12,w6
3310
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
3311
	eor	w6,w14,w15
3312
	eor	w9,w12,w8
3313
	eor	w6,w6,w9
3314
	mov	v3.s[0],w6
3315
	// optimize sbox using AESE instruction
3316
	tbl	v0.16b, {v3.16b}, v26.16b
3317
	ushr	v2.16b, v0.16b, 4
3318
	and	v0.16b, v0.16b, v31.16b
3319
	tbl	v0.16b, {v28.16b}, v0.16b
3320
	tbl	v2.16b, {v27.16b}, v2.16b
3321
	eor	v0.16b, v0.16b, v2.16b
3322
	eor	v1.16b, v1.16b, v1.16b
3323
	aese	v0.16b,v1.16b
3324
	ushr	v2.16b, v0.16b, 4
3325
	and	v0.16b, v0.16b, v31.16b
3326
	tbl	v0.16b, {v30.16b}, v0.16b
3327
	tbl	v2.16b, {v29.16b}, v2.16b
3328
	eor	v0.16b, v0.16b, v2.16b
3329

3330
	mov	w7,v0.s[0]
3331
	eor	w6,w7,w7,ror #32-2
3332
	eor	w6,w6,w7,ror #32-10
3333
	eor	w6,w6,w7,ror #32-18
3334
	eor	w6,w6,w7,ror #32-24
3335
	ldp	w7,w8,[x10],8
3336
	eor	w13,w13,w6
3337
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
3338
	eor	w6,w12,w13
3339
	eor	w9,w7,w15
3340
	eor	w6,w6,w9
3341
	mov	v3.s[0],w6
3342
	// optimize sbox using AESE instruction
3343
	tbl	v0.16b, {v3.16b}, v26.16b
3344
	ushr	v2.16b, v0.16b, 4
3345
	and	v0.16b, v0.16b, v31.16b
3346
	tbl	v0.16b, {v28.16b}, v0.16b
3347
	tbl	v2.16b, {v27.16b}, v2.16b
3348
	eor	v0.16b, v0.16b, v2.16b
3349
	eor	v1.16b, v1.16b, v1.16b
3350
	aese	v0.16b,v1.16b
3351
	ushr	v2.16b, v0.16b, 4
3352
	and	v0.16b, v0.16b, v31.16b
3353
	tbl	v0.16b, {v30.16b}, v0.16b
3354
	tbl	v2.16b, {v29.16b}, v2.16b
3355
	eor	v0.16b, v0.16b, v2.16b
3356

3357
	mov	w7,v0.s[0]
3358
	eor	w6,w7,w7,ror #32-2
3359
	eor	w6,w6,w7,ror #32-10
3360
	eor	w6,w6,w7,ror #32-18
3361
	eor	w6,w6,w7,ror #32-24
3362
	eor	w14,w14,w6
3363
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
3364
	eor	w6,w12,w13
3365
	eor	w9,w14,w8
3366
	eor	w6,w6,w9
3367
	mov	v3.s[0],w6
3368
	// optimize sbox using AESE instruction
3369
	tbl	v0.16b, {v3.16b}, v26.16b
3370
	ushr	v2.16b, v0.16b, 4
3371
	and	v0.16b, v0.16b, v31.16b
3372
	tbl	v0.16b, {v28.16b}, v0.16b
3373
	tbl	v2.16b, {v27.16b}, v2.16b
3374
	eor	v0.16b, v0.16b, v2.16b
3375
	eor	v1.16b, v1.16b, v1.16b
3376
	aese	v0.16b,v1.16b
3377
	ushr	v2.16b, v0.16b, 4
3378
	and	v0.16b, v0.16b, v31.16b
3379
	tbl	v0.16b, {v30.16b}, v0.16b
3380
	tbl	v2.16b, {v29.16b}, v2.16b
3381
	eor	v0.16b, v0.16b, v2.16b
3382

3383
	mov	w7,v0.s[0]
3384
	eor	w6,w7,w7,ror #32-2
3385
	eor	w6,w6,w7,ror #32-10
3386
	eor	w6,w6,w7,ror #32-18
3387
	eor	w6,w6,w7,ror #32-24
3388
	eor	w15,w15,w6
3389
	subs	w11,w11,#1
3390
	b.ne	10b
3391
	mov	v4.s[0],w15
3392
	mov	v4.s[1],w14
3393
	mov	v4.s[2],w13
3394
	mov	v4.s[3],w12
3395
#ifndef __AARCH64EB__
3396
	rev32	v4.16b,v4.16b
3397
#endif
3398
	eor	v4.16b, v4.16b, v17.16b
3399
	st1	{v4.4s},[x1],#16
3400

3401
	sub	x26,x1,16
3402
.loop_gb:
3403
	subs	x29,x29,1
3404
	ldrb	w7,[x26,x29]
3405
	ldrb	w8,[x0,x29]
3406
	strb	w8,[x26,x29]
3407
	strb	w7,[x1,x29]
3408
	b.gt	.loop_gb
3409
	ld1	{v4.4s}, [x26]
3410
	eor	v4.16b, v4.16b, v18.16b
3411
#ifndef __AARCH64EB__
3412
	rev32	v4.16b,v4.16b
3413
#endif
3414
	mov	x10,x3
3415
	mov	w11,#8
3416
	mov	w12,v4.s[0]
3417
	mov	w13,v4.s[1]
3418
	mov	w14,v4.s[2]
3419
	mov	w15,v4.s[3]
3420
10:
3421
	ldp	w7,w8,[x10],8
3422
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
3423
	eor	w6,w14,w15
3424
	eor	w9,w7,w13
3425
	eor	w6,w6,w9
3426
	mov	v3.s[0],w6
3427
	// optimize sbox using AESE instruction
3428
	tbl	v0.16b, {v3.16b}, v26.16b
3429
	ushr	v2.16b, v0.16b, 4
3430
	and	v0.16b, v0.16b, v31.16b
3431
	tbl	v0.16b, {v28.16b}, v0.16b
3432
	tbl	v2.16b, {v27.16b}, v2.16b
3433
	eor	v0.16b, v0.16b, v2.16b
3434
	eor	v1.16b, v1.16b, v1.16b
3435
	aese	v0.16b,v1.16b
3436
	ushr	v2.16b, v0.16b, 4
3437
	and	v0.16b, v0.16b, v31.16b
3438
	tbl	v0.16b, {v30.16b}, v0.16b
3439
	tbl	v2.16b, {v29.16b}, v2.16b
3440
	eor	v0.16b, v0.16b, v2.16b
3441

3442
	mov	w7,v0.s[0]
3443
	eor	w6,w7,w7,ror #32-2
3444
	eor	w6,w6,w7,ror #32-10
3445
	eor	w6,w6,w7,ror #32-18
3446
	eor	w6,w6,w7,ror #32-24
3447
	eor	w12,w12,w6
3448
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
3449
	eor	w6,w14,w15
3450
	eor	w9,w12,w8
3451
	eor	w6,w6,w9
3452
	mov	v3.s[0],w6
3453
	// optimize sbox using AESE instruction
3454
	tbl	v0.16b, {v3.16b}, v26.16b
3455
	ushr	v2.16b, v0.16b, 4
3456
	and	v0.16b, v0.16b, v31.16b
3457
	tbl	v0.16b, {v28.16b}, v0.16b
3458
	tbl	v2.16b, {v27.16b}, v2.16b
3459
	eor	v0.16b, v0.16b, v2.16b
3460
	eor	v1.16b, v1.16b, v1.16b
3461
	aese	v0.16b,v1.16b
3462
	ushr	v2.16b, v0.16b, 4
3463
	and	v0.16b, v0.16b, v31.16b
3464
	tbl	v0.16b, {v30.16b}, v0.16b
3465
	tbl	v2.16b, {v29.16b}, v2.16b
3466
	eor	v0.16b, v0.16b, v2.16b
3467

3468
	mov	w7,v0.s[0]
3469
	eor	w6,w7,w7,ror #32-2
3470
	eor	w6,w6,w7,ror #32-10
3471
	eor	w6,w6,w7,ror #32-18
3472
	eor	w6,w6,w7,ror #32-24
3473
	ldp	w7,w8,[x10],8
3474
	eor	w13,w13,w6
3475
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
3476
	eor	w6,w12,w13
3477
	eor	w9,w7,w15
3478
	eor	w6,w6,w9
3479
	mov	v3.s[0],w6
3480
	// optimize sbox using AESE instruction
3481
	tbl	v0.16b, {v3.16b}, v26.16b
3482
	ushr	v2.16b, v0.16b, 4
3483
	and	v0.16b, v0.16b, v31.16b
3484
	tbl	v0.16b, {v28.16b}, v0.16b
3485
	tbl	v2.16b, {v27.16b}, v2.16b
3486
	eor	v0.16b, v0.16b, v2.16b
3487
	eor	v1.16b, v1.16b, v1.16b
3488
	aese	v0.16b,v1.16b
3489
	ushr	v2.16b, v0.16b, 4
3490
	and	v0.16b, v0.16b, v31.16b
3491
	tbl	v0.16b, {v30.16b}, v0.16b
3492
	tbl	v2.16b, {v29.16b}, v2.16b
3493
	eor	v0.16b, v0.16b, v2.16b
3494

3495
	mov	w7,v0.s[0]
3496
	eor	w6,w7,w7,ror #32-2
3497
	eor	w6,w6,w7,ror #32-10
3498
	eor	w6,w6,w7,ror #32-18
3499
	eor	w6,w6,w7,ror #32-24
3500
	eor	w14,w14,w6
3501
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
3502
	eor	w6,w12,w13
3503
	eor	w9,w14,w8
3504
	eor	w6,w6,w9
3505
	mov	v3.s[0],w6
3506
	// optimize sbox using AESE instruction
3507
	tbl	v0.16b, {v3.16b}, v26.16b
3508
	ushr	v2.16b, v0.16b, 4
3509
	and	v0.16b, v0.16b, v31.16b
3510
	tbl	v0.16b, {v28.16b}, v0.16b
3511
	tbl	v2.16b, {v27.16b}, v2.16b
3512
	eor	v0.16b, v0.16b, v2.16b
3513
	eor	v1.16b, v1.16b, v1.16b
3514
	aese	v0.16b,v1.16b
3515
	ushr	v2.16b, v0.16b, 4
3516
	and	v0.16b, v0.16b, v31.16b
3517
	tbl	v0.16b, {v30.16b}, v0.16b
3518
	tbl	v2.16b, {v29.16b}, v2.16b
3519
	eor	v0.16b, v0.16b, v2.16b
3520

3521
	mov	w7,v0.s[0]
3522
	eor	w6,w7,w7,ror #32-2
3523
	eor	w6,w6,w7,ror #32-10
3524
	eor	w6,w6,w7,ror #32-18
3525
	eor	w6,w6,w7,ror #32-24
3526
	eor	w15,w15,w6
3527
	subs	w11,w11,#1
3528
	b.ne	10b
3529
	mov	v4.s[0],w15
3530
	mov	v4.s[1],w14
3531
	mov	v4.s[2],w13
3532
	mov	v4.s[3],w12
3533
#ifndef __AARCH64EB__
3534
	rev32	v4.16b,v4.16b
3535
#endif
3536
	eor	v4.16b, v4.16b, v18.16b
3537
	st1	{v4.4s}, [x26]
3538
.return_gb:
3539
	ldp	d14, d15, [sp], #0x10
3540
	ldp	d12, d13, [sp], #0x10
3541
	ldp	d10, d11, [sp], #0x10
3542
	ldp	d8, d9, [sp], #0x10
3543
	ldp	x29, x30, [sp], #0x10
3544
	ldp	x27, x28, [sp], #0x10
3545
	ldp	x25, x26, [sp], #0x10
3546
	ldp	x23, x24, [sp], #0x10
3547
	ldp	x21, x22, [sp], #0x10
3548
	ldp	x19, x20, [sp], #0x10
3549
	ldp	x17, x18, [sp], #0x10
3550
	ldp	x15, x16, [sp], #0x10
3551
	AARCH64_VALIDATE_LINK_REGISTER
3552
	ret
3553
.size	vpsm4_ex_xts_encrypt_gb,.-vpsm4_ex_xts_encrypt_gb
3554
.globl	vpsm4_ex_xts_encrypt
3555
.type	vpsm4_ex_xts_encrypt,%function
3556
.align	5
3557
vpsm4_ex_xts_encrypt:
3558
	AARCH64_SIGN_LINK_REGISTER
3559
	stp	x15, x16, [sp, #-0x10]!
3560
	stp	x17, x18, [sp, #-0x10]!
3561
	stp	x19, x20, [sp, #-0x10]!
3562
	stp	x21, x22, [sp, #-0x10]!
3563
	stp	x23, x24, [sp, #-0x10]!
3564
	stp	x25, x26, [sp, #-0x10]!
3565
	stp	x27, x28, [sp, #-0x10]!
3566
	stp	x29, x30, [sp, #-0x10]!
3567
	stp	d8, d9, [sp, #-0x10]!
3568
	stp	d10, d11, [sp, #-0x10]!
3569
	stp	d12, d13, [sp, #-0x10]!
3570
	stp	d14, d15, [sp, #-0x10]!
3571
	mov	x26,x3
3572
	mov	x27,x4
3573
	mov	w28,w6
3574
	ld1	{v16.4s}, [x5]
3575
	mov	x3,x27
3576
	adrp	x9, .Lsbox_magic
3577
	ldr	q26, [x9, #:lo12:.Lsbox_magic]
3578
	ldr	q27, [x9, #:lo12:.Lsbox_magic+16]
3579
	ldr	q28, [x9, #:lo12:.Lsbox_magic+32]
3580
	ldr	q29, [x9, #:lo12:.Lsbox_magic+48]
3581
	ldr	q30, [x9, #:lo12:.Lsbox_magic+64]
3582
	ldr	q31, [x9, #:lo12:.Lsbox_magic+80]
3583
#ifndef __AARCH64EB__
3584
	rev32	v16.16b,v16.16b
3585
#endif
3586
	mov	x10,x3
3587
	mov	w11,#8
3588
	mov	w12,v16.s[0]
3589
	mov	w13,v16.s[1]
3590
	mov	w14,v16.s[2]
3591
	mov	w15,v16.s[3]
3592
10:
3593
	ldp	w7,w8,[x10],8
3594
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
3595
	eor	w6,w14,w15
3596
	eor	w9,w7,w13
3597
	eor	w6,w6,w9
3598
	mov	v3.s[0],w6
3599
	// optimize sbox using AESE instruction
3600
	tbl	v0.16b, {v3.16b}, v26.16b
3601
	ushr	v2.16b, v0.16b, 4
3602
	and	v0.16b, v0.16b, v31.16b
3603
	tbl	v0.16b, {v28.16b}, v0.16b
3604
	tbl	v2.16b, {v27.16b}, v2.16b
3605
	eor	v0.16b, v0.16b, v2.16b
3606
	eor	v1.16b, v1.16b, v1.16b
3607
	aese	v0.16b,v1.16b
3608
	ushr	v2.16b, v0.16b, 4
3609
	and	v0.16b, v0.16b, v31.16b
3610
	tbl	v0.16b, {v30.16b}, v0.16b
3611
	tbl	v2.16b, {v29.16b}, v2.16b
3612
	eor	v0.16b, v0.16b, v2.16b
3613

3614
	mov	w7,v0.s[0]
3615
	eor	w6,w7,w7,ror #32-2
3616
	eor	w6,w6,w7,ror #32-10
3617
	eor	w6,w6,w7,ror #32-18
3618
	eor	w6,w6,w7,ror #32-24
3619
	eor	w12,w12,w6
3620
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
3621
	eor	w6,w14,w15
3622
	eor	w9,w12,w8
3623
	eor	w6,w6,w9
3624
	mov	v3.s[0],w6
3625
	// optimize sbox using AESE instruction
3626
	tbl	v0.16b, {v3.16b}, v26.16b
3627
	ushr	v2.16b, v0.16b, 4
3628
	and	v0.16b, v0.16b, v31.16b
3629
	tbl	v0.16b, {v28.16b}, v0.16b
3630
	tbl	v2.16b, {v27.16b}, v2.16b
3631
	eor	v0.16b, v0.16b, v2.16b
3632
	eor	v1.16b, v1.16b, v1.16b
3633
	aese	v0.16b,v1.16b
3634
	ushr	v2.16b, v0.16b, 4
3635
	and	v0.16b, v0.16b, v31.16b
3636
	tbl	v0.16b, {v30.16b}, v0.16b
3637
	tbl	v2.16b, {v29.16b}, v2.16b
3638
	eor	v0.16b, v0.16b, v2.16b
3639

3640
	mov	w7,v0.s[0]
3641
	eor	w6,w7,w7,ror #32-2
3642
	eor	w6,w6,w7,ror #32-10
3643
	eor	w6,w6,w7,ror #32-18
3644
	eor	w6,w6,w7,ror #32-24
3645
	ldp	w7,w8,[x10],8
3646
	eor	w13,w13,w6
3647
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
3648
	eor	w6,w12,w13
3649
	eor	w9,w7,w15
3650
	eor	w6,w6,w9
3651
	mov	v3.s[0],w6
3652
	// optimize sbox using AESE instruction
3653
	tbl	v0.16b, {v3.16b}, v26.16b
3654
	ushr	v2.16b, v0.16b, 4
3655
	and	v0.16b, v0.16b, v31.16b
3656
	tbl	v0.16b, {v28.16b}, v0.16b
3657
	tbl	v2.16b, {v27.16b}, v2.16b
3658
	eor	v0.16b, v0.16b, v2.16b
3659
	eor	v1.16b, v1.16b, v1.16b
3660
	aese	v0.16b,v1.16b
3661
	ushr	v2.16b, v0.16b, 4
3662
	and	v0.16b, v0.16b, v31.16b
3663
	tbl	v0.16b, {v30.16b}, v0.16b
3664
	tbl	v2.16b, {v29.16b}, v2.16b
3665
	eor	v0.16b, v0.16b, v2.16b
3666

3667
	mov	w7,v0.s[0]
3668
	eor	w6,w7,w7,ror #32-2
3669
	eor	w6,w6,w7,ror #32-10
3670
	eor	w6,w6,w7,ror #32-18
3671
	eor	w6,w6,w7,ror #32-24
3672
	eor	w14,w14,w6
3673
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
3674
	eor	w6,w12,w13
3675
	eor	w9,w14,w8
3676
	eor	w6,w6,w9
3677
	mov	v3.s[0],w6
3678
	// optimize sbox using AESE instruction
3679
	tbl	v0.16b, {v3.16b}, v26.16b
3680
	ushr	v2.16b, v0.16b, 4
3681
	and	v0.16b, v0.16b, v31.16b
3682
	tbl	v0.16b, {v28.16b}, v0.16b
3683
	tbl	v2.16b, {v27.16b}, v2.16b
3684
	eor	v0.16b, v0.16b, v2.16b
3685
	eor	v1.16b, v1.16b, v1.16b
3686
	aese	v0.16b,v1.16b
3687
	ushr	v2.16b, v0.16b, 4
3688
	and	v0.16b, v0.16b, v31.16b
3689
	tbl	v0.16b, {v30.16b}, v0.16b
3690
	tbl	v2.16b, {v29.16b}, v2.16b
3691
	eor	v0.16b, v0.16b, v2.16b
3692

3693
	mov	w7,v0.s[0]
3694
	eor	w6,w7,w7,ror #32-2
3695
	eor	w6,w6,w7,ror #32-10
3696
	eor	w6,w6,w7,ror #32-18
3697
	eor	w6,w6,w7,ror #32-24
3698
	eor	w15,w15,w6
3699
	subs	w11,w11,#1
3700
	b.ne	10b
3701
	mov	v16.s[0],w15
3702
	mov	v16.s[1],w14
3703
	mov	v16.s[2],w13
3704
	mov	v16.s[3],w12
3705
#ifndef __AARCH64EB__
3706
	rev32	v16.16b,v16.16b
3707
#endif
3708
	mov	x3,x26
3709
	and	x29,x2,#0x0F
3710
	// convert length into blocks
3711
	lsr	x2,x2,4
3712
	cmp	x2,#1
3713
	b.lt	.return
3714

3715
	cmp	x29,0
3716
	// If the encryption/decryption Length is N times of 16,
3717
	// the all blocks are encrypted/decrypted in .xts_encrypt_blocks
3718
	b.eq	.xts_encrypt_blocks
3719

3720
	// If the encryption/decryption length is not N times of 16,
3721
	// the last two blocks are encrypted/decrypted in .last_2blks_tweak or .only_2blks_tweak
3722
	// the other blocks are encrypted/decrypted in .xts_encrypt_blocks
3723
	subs	x2,x2,#1
3724
	b.eq	.only_2blks_tweak
3725
.xts_encrypt_blocks:
3726
#ifdef __AARCH64EB__
3727
	rev32	v16.16b,v16.16b
3728
#endif
3729
	mov	x12,v16.d[0]
3730
	mov	x13,v16.d[1]
3731
	mov	w7,0x87
3732
	extr	x9,x13,x13,#32
3733
	extr	x15,x13,x12,#63
3734
	and	w8,w7,w9,asr#31
3735
	eor	x14,x8,x12,lsl#1
3736
	mov	w7,0x87
3737
	extr	x9,x15,x15,#32
3738
	extr	x17,x15,x14,#63
3739
	and	w8,w7,w9,asr#31
3740
	eor	x16,x8,x14,lsl#1
3741
	mov	w7,0x87
3742
	extr	x9,x17,x17,#32
3743
	extr	x19,x17,x16,#63
3744
	and	w8,w7,w9,asr#31
3745
	eor	x18,x8,x16,lsl#1
3746
	mov	w7,0x87
3747
	extr	x9,x19,x19,#32
3748
	extr	x21,x19,x18,#63
3749
	and	w8,w7,w9,asr#31
3750
	eor	x20,x8,x18,lsl#1
3751
	mov	w7,0x87
3752
	extr	x9,x21,x21,#32
3753
	extr	x23,x21,x20,#63
3754
	and	w8,w7,w9,asr#31
3755
	eor	x22,x8,x20,lsl#1
3756
	mov	w7,0x87
3757
	extr	x9,x23,x23,#32
3758
	extr	x25,x23,x22,#63
3759
	and	w8,w7,w9,asr#31
3760
	eor	x24,x8,x22,lsl#1
3761
	mov	w7,0x87
3762
	extr	x9,x25,x25,#32
3763
	extr	x27,x25,x24,#63
3764
	and	w8,w7,w9,asr#31
3765
	eor	x26,x8,x24,lsl#1
3766
.Lxts_8_blocks_process:
3767
	cmp	x2,#8
3768
	mov	v16.d[0],x12
3769
	mov	v16.d[1],x13
3770
#ifdef __AARCH64EB__
3771
	rev32	v16.16b,v16.16b
3772
#endif
3773
	mov	w7,0x87
3774
	extr	x9,x27,x27,#32
3775
	extr	x13,x27,x26,#63
3776
	and	w8,w7,w9,asr#31
3777
	eor	x12,x8,x26,lsl#1
3778
	mov	v17.d[0],x14
3779
	mov	v17.d[1],x15
3780
#ifdef __AARCH64EB__
3781
	rev32	v17.16b,v17.16b
3782
#endif
3783
	mov	w7,0x87
3784
	extr	x9,x13,x13,#32
3785
	extr	x15,x13,x12,#63
3786
	and	w8,w7,w9,asr#31
3787
	eor	x14,x8,x12,lsl#1
3788
	mov	v18.d[0],x16
3789
	mov	v18.d[1],x17
3790
#ifdef __AARCH64EB__
3791
	rev32	v18.16b,v18.16b
3792
#endif
3793
	mov	w7,0x87
3794
	extr	x9,x15,x15,#32
3795
	extr	x17,x15,x14,#63
3796
	and	w8,w7,w9,asr#31
3797
	eor	x16,x8,x14,lsl#1
3798
	mov	v19.d[0],x18
3799
	mov	v19.d[1],x19
3800
#ifdef __AARCH64EB__
3801
	rev32	v19.16b,v19.16b
3802
#endif
3803
	mov	w7,0x87
3804
	extr	x9,x17,x17,#32
3805
	extr	x19,x17,x16,#63
3806
	and	w8,w7,w9,asr#31
3807
	eor	x18,x8,x16,lsl#1
3808
	mov	v20.d[0],x20
3809
	mov	v20.d[1],x21
3810
#ifdef __AARCH64EB__
3811
	rev32	v20.16b,v20.16b
3812
#endif
3813
	mov	w7,0x87
3814
	extr	x9,x19,x19,#32
3815
	extr	x21,x19,x18,#63
3816
	and	w8,w7,w9,asr#31
3817
	eor	x20,x8,x18,lsl#1
3818
	mov	v21.d[0],x22
3819
	mov	v21.d[1],x23
3820
#ifdef __AARCH64EB__
3821
	rev32	v21.16b,v21.16b
3822
#endif
3823
	mov	w7,0x87
3824
	extr	x9,x21,x21,#32
3825
	extr	x23,x21,x20,#63
3826
	and	w8,w7,w9,asr#31
3827
	eor	x22,x8,x20,lsl#1
3828
	mov	v22.d[0],x24
3829
	mov	v22.d[1],x25
3830
#ifdef __AARCH64EB__
3831
	rev32	v22.16b,v22.16b
3832
#endif
3833
	mov	w7,0x87
3834
	extr	x9,x23,x23,#32
3835
	extr	x25,x23,x22,#63
3836
	and	w8,w7,w9,asr#31
3837
	eor	x24,x8,x22,lsl#1
3838
	mov	v23.d[0],x26
3839
	mov	v23.d[1],x27
3840
#ifdef __AARCH64EB__
3841
	rev32	v23.16b,v23.16b
3842
#endif
3843
	mov	w7,0x87
3844
	extr	x9,x25,x25,#32
3845
	extr	x27,x25,x24,#63
3846
	and	w8,w7,w9,asr#31
3847
	eor	x26,x8,x24,lsl#1
3848
	b.lt	.Lxts_4_blocks_process
3849
	ld1	{v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64
3850
	eor	v4.16b, v4.16b, v16.16b
3851
	eor	v5.16b, v5.16b, v17.16b
3852
	eor	v6.16b, v6.16b, v18.16b
3853
	eor	v7.16b, v7.16b, v19.16b
3854
	ld1	{v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64
3855
	eor	v8.16b, v8.16b, v20.16b
3856
	eor	v9.16b, v9.16b, v21.16b
3857
	eor	v10.16b, v10.16b, v22.16b
3858
	eor	v11.16b, v11.16b, v23.16b
3859
#ifndef __AARCH64EB__
3860
	rev32	v4.16b,v4.16b
3861
#endif
3862
#ifndef __AARCH64EB__
3863
	rev32	v5.16b,v5.16b
3864
#endif
3865
#ifndef __AARCH64EB__
3866
	rev32	v6.16b,v6.16b
3867
#endif
3868
#ifndef __AARCH64EB__
3869
	rev32	v7.16b,v7.16b
3870
#endif
3871
#ifndef __AARCH64EB__
3872
	rev32	v8.16b,v8.16b
3873
#endif
3874
#ifndef __AARCH64EB__
3875
	rev32	v9.16b,v9.16b
3876
#endif
3877
#ifndef __AARCH64EB__
3878
	rev32	v10.16b,v10.16b
3879
#endif
3880
#ifndef __AARCH64EB__
3881
	rev32	v11.16b,v11.16b
3882
#endif
3883
	zip1	v0.4s,v4.4s,v5.4s
3884
	zip2	v1.4s,v4.4s,v5.4s
3885
	zip1	v2.4s,v6.4s,v7.4s
3886
	zip2	v3.4s,v6.4s,v7.4s
3887
	zip1	v4.2d,v0.2d,v2.2d
3888
	zip2	v5.2d,v0.2d,v2.2d
3889
	zip1	v6.2d,v1.2d,v3.2d
3890
	zip2	v7.2d,v1.2d,v3.2d
3891
	zip1	v0.4s,v8.4s,v9.4s
3892
	zip2	v1.4s,v8.4s,v9.4s
3893
	zip1	v2.4s,v10.4s,v11.4s
3894
	zip2	v3.4s,v10.4s,v11.4s
3895
	zip1	v8.2d,v0.2d,v2.2d
3896
	zip2	v9.2d,v0.2d,v2.2d
3897
	zip1	v10.2d,v1.2d,v3.2d
3898
	zip2	v11.2d,v1.2d,v3.2d
3899
	bl	_vpsm4_ex_enc_8blks
3900
	zip1	v8.4s,v0.4s,v1.4s
3901
	zip2	v9.4s,v0.4s,v1.4s
3902
	zip1	v10.4s,v2.4s,v3.4s
3903
	zip2	v11.4s,v2.4s,v3.4s
3904
	zip1	v0.2d,v8.2d,v10.2d
3905
	zip2	v1.2d,v8.2d,v10.2d
3906
	zip1	v2.2d,v9.2d,v11.2d
3907
	zip2	v3.2d,v9.2d,v11.2d
3908
	zip1	v8.4s,v4.4s,v5.4s
3909
	zip2	v9.4s,v4.4s,v5.4s
3910
	zip1	v10.4s,v6.4s,v7.4s
3911
	zip2	v11.4s,v6.4s,v7.4s
3912
	zip1	v4.2d,v8.2d,v10.2d
3913
	zip2	v5.2d,v8.2d,v10.2d
3914
	zip1	v6.2d,v9.2d,v11.2d
3915
	zip2	v7.2d,v9.2d,v11.2d
3916
	eor	v0.16b, v0.16b, v16.16b
3917
	eor	v1.16b, v1.16b, v17.16b
3918
	eor	v2.16b, v2.16b, v18.16b
3919
	eor	v3.16b, v3.16b, v19.16b
3920
	eor	v4.16b, v4.16b, v20.16b
3921
	eor	v5.16b, v5.16b, v21.16b
3922
	eor	v6.16b, v6.16b, v22.16b
3923
	eor	v7.16b, v7.16b, v23.16b
3924

3925
	// save the last tweak
3926
	mov	v25.16b,v23.16b
3927
	st1	{v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64
3928
	st1	{v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64
3929
	subs	x2,x2,#8
3930
	b.gt	.Lxts_8_blocks_process
3931
	b	100f
3932
.Lxts_4_blocks_process:
3933
	cmp	x2,#4
3934
	b.lt	1f
3935
	ld1	{v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64
3936
	eor	v4.16b, v4.16b, v16.16b
3937
	eor	v5.16b, v5.16b, v17.16b
3938
	eor	v6.16b, v6.16b, v18.16b
3939
	eor	v7.16b, v7.16b, v19.16b
3940
#ifndef __AARCH64EB__
3941
	rev32	v4.16b,v4.16b
3942
#endif
3943
#ifndef __AARCH64EB__
3944
	rev32	v5.16b,v5.16b
3945
#endif
3946
#ifndef __AARCH64EB__
3947
	rev32	v6.16b,v6.16b
3948
#endif
3949
#ifndef __AARCH64EB__
3950
	rev32	v7.16b,v7.16b
3951
#endif
3952
	zip1	v0.4s,v4.4s,v5.4s
3953
	zip2	v1.4s,v4.4s,v5.4s
3954
	zip1	v2.4s,v6.4s,v7.4s
3955
	zip2	v3.4s,v6.4s,v7.4s
3956
	zip1	v4.2d,v0.2d,v2.2d
3957
	zip2	v5.2d,v0.2d,v2.2d
3958
	zip1	v6.2d,v1.2d,v3.2d
3959
	zip2	v7.2d,v1.2d,v3.2d
3960
	bl	_vpsm4_ex_enc_4blks
3961
	zip1	v4.4s,v0.4s,v1.4s
3962
	zip2	v5.4s,v0.4s,v1.4s
3963
	zip1	v6.4s,v2.4s,v3.4s
3964
	zip2	v7.4s,v2.4s,v3.4s
3965
	zip1	v0.2d,v4.2d,v6.2d
3966
	zip2	v1.2d,v4.2d,v6.2d
3967
	zip1	v2.2d,v5.2d,v7.2d
3968
	zip2	v3.2d,v5.2d,v7.2d
3969
	eor	v0.16b, v0.16b, v16.16b
3970
	eor	v1.16b, v1.16b, v17.16b
3971
	eor	v2.16b, v2.16b, v18.16b
3972
	eor	v3.16b, v3.16b, v19.16b
3973
	st1	{v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64
3974
	sub	x2,x2,#4
3975
	mov	v16.16b,v20.16b
3976
	mov	v17.16b,v21.16b
3977
	mov	v18.16b,v22.16b
3978
	// save the last tweak
3979
	mov	v25.16b,v19.16b
3980
1:
3981
	// process last block
3982
	cmp	x2,#1
3983
	b.lt	100f
3984
	b.gt	1f
3985
	ld1	{v4.4s},[x0],#16
3986
	eor	v4.16b, v4.16b, v16.16b
3987
#ifndef __AARCH64EB__
3988
	rev32	v4.16b,v4.16b
3989
#endif
3990
	mov	x10,x3
3991
	mov	w11,#8
3992
	mov	w12,v4.s[0]
3993
	mov	w13,v4.s[1]
3994
	mov	w14,v4.s[2]
3995
	mov	w15,v4.s[3]
3996
10:
3997
	ldp	w7,w8,[x10],8
3998
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
3999
	eor	w6,w14,w15
4000
	eor	w9,w7,w13
4001
	eor	w6,w6,w9
4002
	mov	v3.s[0],w6
4003
	// optimize sbox using AESE instruction
4004
	tbl	v0.16b, {v3.16b}, v26.16b
4005
	ushr	v2.16b, v0.16b, 4
4006
	and	v0.16b, v0.16b, v31.16b
4007
	tbl	v0.16b, {v28.16b}, v0.16b
4008
	tbl	v2.16b, {v27.16b}, v2.16b
4009
	eor	v0.16b, v0.16b, v2.16b
4010
	eor	v1.16b, v1.16b, v1.16b
4011
	aese	v0.16b,v1.16b
4012
	ushr	v2.16b, v0.16b, 4
4013
	and	v0.16b, v0.16b, v31.16b
4014
	tbl	v0.16b, {v30.16b}, v0.16b
4015
	tbl	v2.16b, {v29.16b}, v2.16b
4016
	eor	v0.16b, v0.16b, v2.16b
4017

4018
	mov	w7,v0.s[0]
4019
	eor	w6,w7,w7,ror #32-2
4020
	eor	w6,w6,w7,ror #32-10
4021
	eor	w6,w6,w7,ror #32-18
4022
	eor	w6,w6,w7,ror #32-24
4023
	eor	w12,w12,w6
4024
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
4025
	eor	w6,w14,w15
4026
	eor	w9,w12,w8
4027
	eor	w6,w6,w9
4028
	mov	v3.s[0],w6
4029
	// optimize sbox using AESE instruction
4030
	tbl	v0.16b, {v3.16b}, v26.16b
4031
	ushr	v2.16b, v0.16b, 4
4032
	and	v0.16b, v0.16b, v31.16b
4033
	tbl	v0.16b, {v28.16b}, v0.16b
4034
	tbl	v2.16b, {v27.16b}, v2.16b
4035
	eor	v0.16b, v0.16b, v2.16b
4036
	eor	v1.16b, v1.16b, v1.16b
4037
	aese	v0.16b,v1.16b
4038
	ushr	v2.16b, v0.16b, 4
4039
	and	v0.16b, v0.16b, v31.16b
4040
	tbl	v0.16b, {v30.16b}, v0.16b
4041
	tbl	v2.16b, {v29.16b}, v2.16b
4042
	eor	v0.16b, v0.16b, v2.16b
4043

4044
	mov	w7,v0.s[0]
4045
	eor	w6,w7,w7,ror #32-2
4046
	eor	w6,w6,w7,ror #32-10
4047
	eor	w6,w6,w7,ror #32-18
4048
	eor	w6,w6,w7,ror #32-24
4049
	ldp	w7,w8,[x10],8
4050
	eor	w13,w13,w6
4051
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
4052
	eor	w6,w12,w13
4053
	eor	w9,w7,w15
4054
	eor	w6,w6,w9
4055
	mov	v3.s[0],w6
4056
	// optimize sbox using AESE instruction
4057
	tbl	v0.16b, {v3.16b}, v26.16b
4058
	ushr	v2.16b, v0.16b, 4
4059
	and	v0.16b, v0.16b, v31.16b
4060
	tbl	v0.16b, {v28.16b}, v0.16b
4061
	tbl	v2.16b, {v27.16b}, v2.16b
4062
	eor	v0.16b, v0.16b, v2.16b
4063
	eor	v1.16b, v1.16b, v1.16b
4064
	aese	v0.16b,v1.16b
4065
	ushr	v2.16b, v0.16b, 4
4066
	and	v0.16b, v0.16b, v31.16b
4067
	tbl	v0.16b, {v30.16b}, v0.16b
4068
	tbl	v2.16b, {v29.16b}, v2.16b
4069
	eor	v0.16b, v0.16b, v2.16b
4070

4071
	mov	w7,v0.s[0]
4072
	eor	w6,w7,w7,ror #32-2
4073
	eor	w6,w6,w7,ror #32-10
4074
	eor	w6,w6,w7,ror #32-18
4075
	eor	w6,w6,w7,ror #32-24
4076
	eor	w14,w14,w6
4077
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
4078
	eor	w6,w12,w13
4079
	eor	w9,w14,w8
4080
	eor	w6,w6,w9
4081
	mov	v3.s[0],w6
4082
	// optimize sbox using AESE instruction
4083
	tbl	v0.16b, {v3.16b}, v26.16b
4084
	ushr	v2.16b, v0.16b, 4
4085
	and	v0.16b, v0.16b, v31.16b
4086
	tbl	v0.16b, {v28.16b}, v0.16b
4087
	tbl	v2.16b, {v27.16b}, v2.16b
4088
	eor	v0.16b, v0.16b, v2.16b
4089
	eor	v1.16b, v1.16b, v1.16b
4090
	aese	v0.16b,v1.16b
4091
	ushr	v2.16b, v0.16b, 4
4092
	and	v0.16b, v0.16b, v31.16b
4093
	tbl	v0.16b, {v30.16b}, v0.16b
4094
	tbl	v2.16b, {v29.16b}, v2.16b
4095
	eor	v0.16b, v0.16b, v2.16b
4096

4097
	mov	w7,v0.s[0]
4098
	eor	w6,w7,w7,ror #32-2
4099
	eor	w6,w6,w7,ror #32-10
4100
	eor	w6,w6,w7,ror #32-18
4101
	eor	w6,w6,w7,ror #32-24
4102
	eor	w15,w15,w6
4103
	subs	w11,w11,#1
4104
	b.ne	10b
4105
	mov	v4.s[0],w15
4106
	mov	v4.s[1],w14
4107
	mov	v4.s[2],w13
4108
	mov	v4.s[3],w12
4109
#ifndef __AARCH64EB__
4110
	rev32	v4.16b,v4.16b
4111
#endif
4112
	eor	v4.16b, v4.16b, v16.16b
4113
	st1	{v4.4s},[x1],#16
4114
	// save the last tweak
4115
	mov	v25.16b,v16.16b
4116
	b	100f
4117
1:	//	process last 2 blocks
4118
	cmp	x2,#2
4119
	b.gt	1f
4120
	ld1	{v4.4s,v5.4s},[x0],#32
4121
	eor	v4.16b, v4.16b, v16.16b
4122
	eor	v5.16b, v5.16b, v17.16b
4123
#ifndef __AARCH64EB__
4124
	rev32	v4.16b,v4.16b
4125
#endif
4126
#ifndef __AARCH64EB__
4127
	rev32	v5.16b,v5.16b
4128
#endif
4129
	zip1	v0.4s,v4.4s,v5.4s
4130
	zip2	v1.4s,v4.4s,v5.4s
4131
	zip1	v2.4s,v6.4s,v7.4s
4132
	zip2	v3.4s,v6.4s,v7.4s
4133
	zip1	v4.2d,v0.2d,v2.2d
4134
	zip2	v5.2d,v0.2d,v2.2d
4135
	zip1	v6.2d,v1.2d,v3.2d
4136
	zip2	v7.2d,v1.2d,v3.2d
4137
	bl	_vpsm4_ex_enc_4blks
4138
	zip1	v4.4s,v0.4s,v1.4s
4139
	zip2	v5.4s,v0.4s,v1.4s
4140
	zip1	v6.4s,v2.4s,v3.4s
4141
	zip2	v7.4s,v2.4s,v3.4s
4142
	zip1	v0.2d,v4.2d,v6.2d
4143
	zip2	v1.2d,v4.2d,v6.2d
4144
	zip1	v2.2d,v5.2d,v7.2d
4145
	zip2	v3.2d,v5.2d,v7.2d
4146
	eor	v0.16b, v0.16b, v16.16b
4147
	eor	v1.16b, v1.16b, v17.16b
4148
	st1	{v0.4s,v1.4s},[x1],#32
4149
	// save the last tweak
4150
	mov	v25.16b,v17.16b
4151
	b	100f
4152
1:	//	process last 3 blocks
4153
	ld1	{v4.4s,v5.4s,v6.4s},[x0],#48
4154
	eor	v4.16b, v4.16b, v16.16b
4155
	eor	v5.16b, v5.16b, v17.16b
4156
	eor	v6.16b, v6.16b, v18.16b
4157
#ifndef __AARCH64EB__
4158
	rev32	v4.16b,v4.16b
4159
#endif
4160
#ifndef __AARCH64EB__
4161
	rev32	v5.16b,v5.16b
4162
#endif
4163
#ifndef __AARCH64EB__
4164
	rev32	v6.16b,v6.16b
4165
#endif
4166
	zip1	v0.4s,v4.4s,v5.4s
4167
	zip2	v1.4s,v4.4s,v5.4s
4168
	zip1	v2.4s,v6.4s,v7.4s
4169
	zip2	v3.4s,v6.4s,v7.4s
4170
	zip1	v4.2d,v0.2d,v2.2d
4171
	zip2	v5.2d,v0.2d,v2.2d
4172
	zip1	v6.2d,v1.2d,v3.2d
4173
	zip2	v7.2d,v1.2d,v3.2d
4174
	bl	_vpsm4_ex_enc_4blks
4175
	zip1	v4.4s,v0.4s,v1.4s
4176
	zip2	v5.4s,v0.4s,v1.4s
4177
	zip1	v6.4s,v2.4s,v3.4s
4178
	zip2	v7.4s,v2.4s,v3.4s
4179
	zip1	v0.2d,v4.2d,v6.2d
4180
	zip2	v1.2d,v4.2d,v6.2d
4181
	zip1	v2.2d,v5.2d,v7.2d
4182
	zip2	v3.2d,v5.2d,v7.2d
4183
	eor	v0.16b, v0.16b, v16.16b
4184
	eor	v1.16b, v1.16b, v17.16b
4185
	eor	v2.16b, v2.16b, v18.16b
4186
	st1	{v0.4s,v1.4s,v2.4s},[x1],#48
4187
	// save the last tweak
4188
	mov	v25.16b,v18.16b
4189
100:
4190
	cmp	x29,0
4191
	b.eq	.return
4192

4193
// This branch calculates the last two tweaks, 
4194
// while the encryption/decryption length is larger than 32
4195
.last_2blks_tweak:
4196
#ifdef __AARCH64EB__
4197
	rev32	v25.16b,v25.16b
4198
#endif
4199
	mov	v2.16b,v25.16b
4200
	adrp	x9, .Lxts_magic
4201
	ldr	q0, [x9, #:lo12:.Lxts_magic]
4202
	shl	v17.16b, v2.16b, #1
4203
	ext	v1.16b, v2.16b, v2.16b,#15
4204
	ushr	v1.16b, v1.16b, #7
4205
	mul	v1.16b, v1.16b, v0.16b
4206
	eor	v17.16b, v17.16b, v1.16b
4207
	mov	v2.16b,v17.16b
4208
	adrp	x9, .Lxts_magic
4209
	ldr	q0, [x9, #:lo12:.Lxts_magic]
4210
	shl	v18.16b, v2.16b, #1
4211
	ext	v1.16b, v2.16b, v2.16b,#15
4212
	ushr	v1.16b, v1.16b, #7
4213
	mul	v1.16b, v1.16b, v0.16b
4214
	eor	v18.16b, v18.16b, v1.16b
4215
	b	.check_dec
4216

4217

4218
// This branch calculates the last two tweaks, 
4219
// while the encryption/decryption length is equal to 32, who only need two tweaks
4220
.only_2blks_tweak:
4221
	mov	v17.16b,v16.16b
4222
#ifdef __AARCH64EB__
4223
	rev32	v17.16b,v17.16b
4224
#endif
4225
	mov	v2.16b,v17.16b
4226
	adrp	x9, .Lxts_magic
4227
	ldr	q0, [x9, #:lo12:.Lxts_magic]
4228
	shl	v18.16b, v2.16b, #1
4229
	ext	v1.16b, v2.16b, v2.16b,#15
4230
	ushr	v1.16b, v1.16b, #7
4231
	mul	v1.16b, v1.16b, v0.16b
4232
	eor	v18.16b, v18.16b, v1.16b
4233
	b	.check_dec
4234

4235

4236
// Determine whether encryption or decryption is required.
4237
// The last two tweaks need to be swapped for decryption.
4238
.check_dec:
4239
	// encryption:1 decryption:0
4240
	cmp	w28,1
4241
	b.eq	.process_last_2blks
4242
	mov	v0.16B,v17.16b
4243
	mov	v17.16B,v18.16b
4244
	mov	v18.16B,v0.16b
4245

4246
.process_last_2blks:
4247
#ifdef __AARCH64EB__
4248
	rev32	v17.16b,v17.16b
4249
#endif
4250
#ifdef __AARCH64EB__
4251
	rev32	v18.16b,v18.16b
4252
#endif
4253
	ld1	{v4.4s},[x0],#16
4254
	eor	v4.16b, v4.16b, v17.16b
4255
#ifndef __AARCH64EB__
4256
	rev32	v4.16b,v4.16b
4257
#endif
4258
	mov	x10,x3
4259
	mov	w11,#8
4260
	mov	w12,v4.s[0]
4261
	mov	w13,v4.s[1]
4262
	mov	w14,v4.s[2]
4263
	mov	w15,v4.s[3]
4264
10:
4265
	ldp	w7,w8,[x10],8
4266
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
4267
	eor	w6,w14,w15
4268
	eor	w9,w7,w13
4269
	eor	w6,w6,w9
4270
	mov	v3.s[0],w6
4271
	// optimize sbox using AESE instruction
4272
	tbl	v0.16b, {v3.16b}, v26.16b
4273
	ushr	v2.16b, v0.16b, 4
4274
	and	v0.16b, v0.16b, v31.16b
4275
	tbl	v0.16b, {v28.16b}, v0.16b
4276
	tbl	v2.16b, {v27.16b}, v2.16b
4277
	eor	v0.16b, v0.16b, v2.16b
4278
	eor	v1.16b, v1.16b, v1.16b
4279
	aese	v0.16b,v1.16b
4280
	ushr	v2.16b, v0.16b, 4
4281
	and	v0.16b, v0.16b, v31.16b
4282
	tbl	v0.16b, {v30.16b}, v0.16b
4283
	tbl	v2.16b, {v29.16b}, v2.16b
4284
	eor	v0.16b, v0.16b, v2.16b
4285

4286
	mov	w7,v0.s[0]
4287
	eor	w6,w7,w7,ror #32-2
4288
	eor	w6,w6,w7,ror #32-10
4289
	eor	w6,w6,w7,ror #32-18
4290
	eor	w6,w6,w7,ror #32-24
4291
	eor	w12,w12,w6
4292
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
4293
	eor	w6,w14,w15
4294
	eor	w9,w12,w8
4295
	eor	w6,w6,w9
4296
	mov	v3.s[0],w6
4297
	// optimize sbox using AESE instruction
4298
	tbl	v0.16b, {v3.16b}, v26.16b
4299
	ushr	v2.16b, v0.16b, 4
4300
	and	v0.16b, v0.16b, v31.16b
4301
	tbl	v0.16b, {v28.16b}, v0.16b
4302
	tbl	v2.16b, {v27.16b}, v2.16b
4303
	eor	v0.16b, v0.16b, v2.16b
4304
	eor	v1.16b, v1.16b, v1.16b
4305
	aese	v0.16b,v1.16b
4306
	ushr	v2.16b, v0.16b, 4
4307
	and	v0.16b, v0.16b, v31.16b
4308
	tbl	v0.16b, {v30.16b}, v0.16b
4309
	tbl	v2.16b, {v29.16b}, v2.16b
4310
	eor	v0.16b, v0.16b, v2.16b
4311

4312
	mov	w7,v0.s[0]
4313
	eor	w6,w7,w7,ror #32-2
4314
	eor	w6,w6,w7,ror #32-10
4315
	eor	w6,w6,w7,ror #32-18
4316
	eor	w6,w6,w7,ror #32-24
4317
	ldp	w7,w8,[x10],8
4318
	eor	w13,w13,w6
4319
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
4320
	eor	w6,w12,w13
4321
	eor	w9,w7,w15
4322
	eor	w6,w6,w9
4323
	mov	v3.s[0],w6
4324
	// optimize sbox using AESE instruction
4325
	tbl	v0.16b, {v3.16b}, v26.16b
4326
	ushr	v2.16b, v0.16b, 4
4327
	and	v0.16b, v0.16b, v31.16b
4328
	tbl	v0.16b, {v28.16b}, v0.16b
4329
	tbl	v2.16b, {v27.16b}, v2.16b
4330
	eor	v0.16b, v0.16b, v2.16b
4331
	eor	v1.16b, v1.16b, v1.16b
4332
	aese	v0.16b,v1.16b
4333
	ushr	v2.16b, v0.16b, 4
4334
	and	v0.16b, v0.16b, v31.16b
4335
	tbl	v0.16b, {v30.16b}, v0.16b
4336
	tbl	v2.16b, {v29.16b}, v2.16b
4337
	eor	v0.16b, v0.16b, v2.16b
4338

4339
	mov	w7,v0.s[0]
4340
	eor	w6,w7,w7,ror #32-2
4341
	eor	w6,w6,w7,ror #32-10
4342
	eor	w6,w6,w7,ror #32-18
4343
	eor	w6,w6,w7,ror #32-24
4344
	eor	w14,w14,w6
4345
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
4346
	eor	w6,w12,w13
4347
	eor	w9,w14,w8
4348
	eor	w6,w6,w9
4349
	mov	v3.s[0],w6
4350
	// optimize sbox using AESE instruction
4351
	tbl	v0.16b, {v3.16b}, v26.16b
4352
	ushr	v2.16b, v0.16b, 4
4353
	and	v0.16b, v0.16b, v31.16b
4354
	tbl	v0.16b, {v28.16b}, v0.16b
4355
	tbl	v2.16b, {v27.16b}, v2.16b
4356
	eor	v0.16b, v0.16b, v2.16b
4357
	eor	v1.16b, v1.16b, v1.16b
4358
	aese	v0.16b,v1.16b
4359
	ushr	v2.16b, v0.16b, 4
4360
	and	v0.16b, v0.16b, v31.16b
4361
	tbl	v0.16b, {v30.16b}, v0.16b
4362
	tbl	v2.16b, {v29.16b}, v2.16b
4363
	eor	v0.16b, v0.16b, v2.16b
4364

4365
	mov	w7,v0.s[0]
4366
	eor	w6,w7,w7,ror #32-2
4367
	eor	w6,w6,w7,ror #32-10
4368
	eor	w6,w6,w7,ror #32-18
4369
	eor	w6,w6,w7,ror #32-24
4370
	eor	w15,w15,w6
4371
	subs	w11,w11,#1
4372
	b.ne	10b
4373
	mov	v4.s[0],w15
4374
	mov	v4.s[1],w14
4375
	mov	v4.s[2],w13
4376
	mov	v4.s[3],w12
4377
#ifndef __AARCH64EB__
4378
	rev32	v4.16b,v4.16b
4379
#endif
4380
	eor	v4.16b, v4.16b, v17.16b
4381
	st1	{v4.4s},[x1],#16
4382

4383
	sub	x26,x1,16
4384
.loop:
4385
	subs	x29,x29,1
4386
	ldrb	w7,[x26,x29]
4387
	ldrb	w8,[x0,x29]
4388
	strb	w8,[x26,x29]
4389
	strb	w7,[x1,x29]
4390
	b.gt	.loop
4391
	ld1	{v4.4s}, [x26]
4392
	eor	v4.16b, v4.16b, v18.16b
4393
#ifndef __AARCH64EB__
4394
	rev32	v4.16b,v4.16b
4395
#endif
4396
	mov	x10,x3
4397
	mov	w11,#8
4398
	mov	w12,v4.s[0]
4399
	mov	w13,v4.s[1]
4400
	mov	w14,v4.s[2]
4401
	mov	w15,v4.s[3]
4402
10:
4403
	ldp	w7,w8,[x10],8
4404
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
4405
	eor	w6,w14,w15
4406
	eor	w9,w7,w13
4407
	eor	w6,w6,w9
4408
	mov	v3.s[0],w6
4409
	// optimize sbox using AESE instruction
4410
	tbl	v0.16b, {v3.16b}, v26.16b
4411
	ushr	v2.16b, v0.16b, 4
4412
	and	v0.16b, v0.16b, v31.16b
4413
	tbl	v0.16b, {v28.16b}, v0.16b
4414
	tbl	v2.16b, {v27.16b}, v2.16b
4415
	eor	v0.16b, v0.16b, v2.16b
4416
	eor	v1.16b, v1.16b, v1.16b
4417
	aese	v0.16b,v1.16b
4418
	ushr	v2.16b, v0.16b, 4
4419
	and	v0.16b, v0.16b, v31.16b
4420
	tbl	v0.16b, {v30.16b}, v0.16b
4421
	tbl	v2.16b, {v29.16b}, v2.16b
4422
	eor	v0.16b, v0.16b, v2.16b
4423

4424
	mov	w7,v0.s[0]
4425
	eor	w6,w7,w7,ror #32-2
4426
	eor	w6,w6,w7,ror #32-10
4427
	eor	w6,w6,w7,ror #32-18
4428
	eor	w6,w6,w7,ror #32-24
4429
	eor	w12,w12,w6
4430
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
4431
	eor	w6,w14,w15
4432
	eor	w9,w12,w8
4433
	eor	w6,w6,w9
4434
	mov	v3.s[0],w6
4435
	// optimize sbox using AESE instruction
4436
	tbl	v0.16b, {v3.16b}, v26.16b
4437
	ushr	v2.16b, v0.16b, 4
4438
	and	v0.16b, v0.16b, v31.16b
4439
	tbl	v0.16b, {v28.16b}, v0.16b
4440
	tbl	v2.16b, {v27.16b}, v2.16b
4441
	eor	v0.16b, v0.16b, v2.16b
4442
	eor	v1.16b, v1.16b, v1.16b
4443
	aese	v0.16b,v1.16b
4444
	ushr	v2.16b, v0.16b, 4
4445
	and	v0.16b, v0.16b, v31.16b
4446
	tbl	v0.16b, {v30.16b}, v0.16b
4447
	tbl	v2.16b, {v29.16b}, v2.16b
4448
	eor	v0.16b, v0.16b, v2.16b
4449

4450
	mov	w7,v0.s[0]
4451
	eor	w6,w7,w7,ror #32-2
4452
	eor	w6,w6,w7,ror #32-10
4453
	eor	w6,w6,w7,ror #32-18
4454
	eor	w6,w6,w7,ror #32-24
4455
	ldp	w7,w8,[x10],8
4456
	eor	w13,w13,w6
4457
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
4458
	eor	w6,w12,w13
4459
	eor	w9,w7,w15
4460
	eor	w6,w6,w9
4461
	mov	v3.s[0],w6
4462
	// optimize sbox using AESE instruction
4463
	tbl	v0.16b, {v3.16b}, v26.16b
4464
	ushr	v2.16b, v0.16b, 4
4465
	and	v0.16b, v0.16b, v31.16b
4466
	tbl	v0.16b, {v28.16b}, v0.16b
4467
	tbl	v2.16b, {v27.16b}, v2.16b
4468
	eor	v0.16b, v0.16b, v2.16b
4469
	eor	v1.16b, v1.16b, v1.16b
4470
	aese	v0.16b,v1.16b
4471
	ushr	v2.16b, v0.16b, 4
4472
	and	v0.16b, v0.16b, v31.16b
4473
	tbl	v0.16b, {v30.16b}, v0.16b
4474
	tbl	v2.16b, {v29.16b}, v2.16b
4475
	eor	v0.16b, v0.16b, v2.16b
4476

4477
	mov	w7,v0.s[0]
4478
	eor	w6,w7,w7,ror #32-2
4479
	eor	w6,w6,w7,ror #32-10
4480
	eor	w6,w6,w7,ror #32-18
4481
	eor	w6,w6,w7,ror #32-24
4482
	eor	w14,w14,w6
4483
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
4484
	eor	w6,w12,w13
4485
	eor	w9,w14,w8
4486
	eor	w6,w6,w9
4487
	mov	v3.s[0],w6
4488
	// optimize sbox using AESE instruction
4489
	tbl	v0.16b, {v3.16b}, v26.16b
4490
	ushr	v2.16b, v0.16b, 4
4491
	and	v0.16b, v0.16b, v31.16b
4492
	tbl	v0.16b, {v28.16b}, v0.16b
4493
	tbl	v2.16b, {v27.16b}, v2.16b
4494
	eor	v0.16b, v0.16b, v2.16b
4495
	eor	v1.16b, v1.16b, v1.16b
4496
	aese	v0.16b,v1.16b
4497
	ushr	v2.16b, v0.16b, 4
4498
	and	v0.16b, v0.16b, v31.16b
4499
	tbl	v0.16b, {v30.16b}, v0.16b
4500
	tbl	v2.16b, {v29.16b}, v2.16b
4501
	eor	v0.16b, v0.16b, v2.16b
4502

4503
	mov	w7,v0.s[0]
4504
	eor	w6,w7,w7,ror #32-2
4505
	eor	w6,w6,w7,ror #32-10
4506
	eor	w6,w6,w7,ror #32-18
4507
	eor	w6,w6,w7,ror #32-24
4508
	eor	w15,w15,w6
4509
	subs	w11,w11,#1
4510
	b.ne	10b
4511
	mov	v4.s[0],w15
4512
	mov	v4.s[1],w14
4513
	mov	v4.s[2],w13
4514
	mov	v4.s[3],w12
4515
#ifndef __AARCH64EB__
4516
	rev32	v4.16b,v4.16b
4517
#endif
4518
	eor	v4.16b, v4.16b, v18.16b
4519
	st1	{v4.4s}, [x26]
4520
.return:
4521
	ldp	d14, d15, [sp], #0x10
4522
	ldp	d12, d13, [sp], #0x10
4523
	ldp	d10, d11, [sp], #0x10
4524
	ldp	d8, d9, [sp], #0x10
4525
	ldp	x29, x30, [sp], #0x10
4526
	ldp	x27, x28, [sp], #0x10
4527
	ldp	x25, x26, [sp], #0x10
4528
	ldp	x23, x24, [sp], #0x10
4529
	ldp	x21, x22, [sp], #0x10
4530
	ldp	x19, x20, [sp], #0x10
4531
	ldp	x17, x18, [sp], #0x10
4532
	ldp	x15, x16, [sp], #0x10
4533
	AARCH64_VALIDATE_LINK_REGISTER
4534
	ret
4535
.size	vpsm4_ex_xts_encrypt,.-vpsm4_ex_xts_encrypt
4536

4537
Product

Resources

Company
