CoCalc -- bsaes-armv8.S

GitHub Repository: freebsd/freebsd-src
Path: blob/main/sys/crypto/openssl/aarch64/bsaes-armv8.S
³⁹⁵⁰⁷ views
1
/* Do not modify. This file is auto-generated from bsaes-armv8.pl. */
2
// Copyright 2021-2025 The OpenSSL Project Authors. All Rights Reserved.
3
//
4
// Licensed under the OpenSSL license (the "License").  You may not use
5
// this file except in compliance with the License.  You can obtain a copy
6
// in the file LICENSE in the source distribution or at
7
// https://www.openssl.org/source/license.html
8
//
9
// ====================================================================
10
// Written by Ben Avison <bavison@riscosopen.org> for the OpenSSL
11
// project. Rights for redistribution and usage in source and binary
12
// forms are granted according to the OpenSSL license.
13
// ====================================================================
14
//
15
// This implementation is a translation of bsaes-armv7 for AArch64.
16
// No attempt has been made to carry across the build switches for
17
// kernel targets, since the Linux kernel crypto support has moved on
18
// from when it was based on OpenSSL.
19

20
// A lot of hand-scheduling has been performed. Consequently, this code
21
// doesn't factor out neatly into macros in the same way that the
22
// AArch32 version did, and there is little to be gained by wrapping it
23
// up in Perl, and it is presented as pure assembly.
24

25

26
#include "crypto/arm_arch.h"
27

28
.text
29

30

31

32

33

34
.type	_bsaes_decrypt8,%function
35
.align	4
36
// On entry:
37
//   x9 -> key (previously expanded using _bsaes_key_convert)
38
//   x10 = number of rounds
39
//   v0-v7 input data
40
// On exit:
41
//   x9-x11 corrupted
42
//   other general-purpose registers preserved
43
//   v0-v7 output data
44
//   v11-v15 preserved
45
//   other SIMD registers corrupted
46
_bsaes_decrypt8:
47
	ldr	q8, [x9], #16
48
	adrp	x11, .LM0ISR
49
	add	x11, x11, #:lo12:.LM0ISR
50
	movi	v9.16b, #0x55
51
	ldr	q10, [x11], #16
52
	movi	v16.16b, #0x33
53
	movi	v17.16b, #0x0f
54
	sub	x10, x10, #1
55
	eor	v0.16b, v0.16b, v8.16b
56
	eor	v1.16b, v1.16b, v8.16b
57
	eor	v2.16b, v2.16b, v8.16b
58
	eor	v4.16b, v4.16b, v8.16b
59
	eor	v3.16b, v3.16b, v8.16b
60
	eor	v5.16b, v5.16b, v8.16b
61
	tbl	v0.16b, {v0.16b}, v10.16b
62
	tbl	v1.16b, {v1.16b}, v10.16b
63
	tbl	v2.16b, {v2.16b}, v10.16b
64
	tbl	v4.16b, {v4.16b}, v10.16b
65
	eor	v6.16b, v6.16b, v8.16b
66
	eor	v7.16b, v7.16b, v8.16b
67
	tbl	v3.16b, {v3.16b}, v10.16b
68
	tbl	v5.16b, {v5.16b}, v10.16b
69
	tbl	v6.16b, {v6.16b}, v10.16b
70
	ushr	v8.2d, v0.2d, #1
71
	tbl	v7.16b, {v7.16b}, v10.16b
72
	ushr	v10.2d, v4.2d, #1
73
	ushr	v18.2d, v2.2d, #1
74
	eor	v8.16b, v8.16b, v1.16b
75
	ushr	v19.2d, v6.2d, #1
76
	eor	v10.16b, v10.16b, v5.16b
77
	eor	v18.16b, v18.16b, v3.16b
78
	and	v8.16b, v8.16b, v9.16b
79
	eor	v19.16b, v19.16b, v7.16b
80
	and	v10.16b, v10.16b, v9.16b
81
	and	v18.16b, v18.16b, v9.16b
82
	eor	v1.16b, v1.16b, v8.16b
83
	shl	v8.2d, v8.2d, #1
84
	and	v9.16b, v19.16b, v9.16b
85
	eor	v5.16b, v5.16b, v10.16b
86
	shl	v10.2d, v10.2d, #1
87
	eor	v3.16b, v3.16b, v18.16b
88
	shl	v18.2d, v18.2d, #1
89
	eor	v0.16b, v0.16b, v8.16b
90
	shl	v8.2d, v9.2d, #1
91
	eor	v7.16b, v7.16b, v9.16b
92
	eor	v4.16b, v4.16b, v10.16b
93
	eor	v2.16b, v2.16b, v18.16b
94
	ushr	v9.2d, v1.2d, #2
95
	eor	v6.16b, v6.16b, v8.16b
96
	ushr	v8.2d, v0.2d, #2
97
	ushr	v10.2d, v5.2d, #2
98
	ushr	v18.2d, v4.2d, #2
99
	eor	v9.16b, v9.16b, v3.16b
100
	eor	v8.16b, v8.16b, v2.16b
101
	eor	v10.16b, v10.16b, v7.16b
102
	eor	v18.16b, v18.16b, v6.16b
103
	and	v9.16b, v9.16b, v16.16b
104
	and	v8.16b, v8.16b, v16.16b
105
	and	v10.16b, v10.16b, v16.16b
106
	and	v16.16b, v18.16b, v16.16b
107
	eor	v3.16b, v3.16b, v9.16b
108
	shl	v9.2d, v9.2d, #2
109
	eor	v2.16b, v2.16b, v8.16b
110
	shl	v8.2d, v8.2d, #2
111
	eor	v7.16b, v7.16b, v10.16b
112
	shl	v10.2d, v10.2d, #2
113
	eor	v6.16b, v6.16b, v16.16b
114
	shl	v16.2d, v16.2d, #2
115
	eor	v1.16b, v1.16b, v9.16b
116
	eor	v0.16b, v0.16b, v8.16b
117
	eor	v5.16b, v5.16b, v10.16b
118
	eor	v4.16b, v4.16b, v16.16b
119
	ushr	v8.2d, v3.2d, #4
120
	ushr	v9.2d, v2.2d, #4
121
	ushr	v10.2d, v1.2d, #4
122
	ushr	v16.2d, v0.2d, #4
123
	eor	v8.16b, v8.16b, v7.16b
124
	eor	v9.16b, v9.16b, v6.16b
125
	eor	v10.16b, v10.16b, v5.16b
126
	eor	v16.16b, v16.16b, v4.16b
127
	and	v8.16b, v8.16b, v17.16b
128
	and	v9.16b, v9.16b, v17.16b
129
	and	v10.16b, v10.16b, v17.16b
130
	and	v16.16b, v16.16b, v17.16b
131
	eor	v7.16b, v7.16b, v8.16b
132
	shl	v8.2d, v8.2d, #4
133
	eor	v6.16b, v6.16b, v9.16b
134
	shl	v9.2d, v9.2d, #4
135
	eor	v5.16b, v5.16b, v10.16b
136
	shl	v10.2d, v10.2d, #4
137
	eor	v4.16b, v4.16b, v16.16b
138
	shl	v16.2d, v16.2d, #4
139
	eor	v3.16b, v3.16b, v8.16b
140
	eor	v2.16b, v2.16b, v9.16b
141
	eor	v1.16b, v1.16b, v10.16b
142
	eor	v0.16b, v0.16b, v16.16b
143
	b	.Ldec_sbox
144
.align	4
145
.Ldec_loop:
146
	ld1	{v16.16b, v17.16b, v18.16b, v19.16b}, [x9], #64
147
	ldp	q8, q9, [x9], #32
148
	eor	v0.16b, v16.16b, v0.16b
149
	ldr	q10, [x9], #16
150
	eor	v1.16b, v17.16b, v1.16b
151
	ldr	q16, [x9], #16
152
	eor	v2.16b, v18.16b, v2.16b
153
	eor	v3.16b, v19.16b, v3.16b
154
	eor	v4.16b, v8.16b, v4.16b
155
	eor	v5.16b, v9.16b, v5.16b
156
	eor	v6.16b, v10.16b, v6.16b
157
	eor	v7.16b, v16.16b, v7.16b
158
	tbl	v0.16b, {v0.16b}, v28.16b
159
	tbl	v1.16b, {v1.16b}, v28.16b
160
	tbl	v2.16b, {v2.16b}, v28.16b
161
	tbl	v3.16b, {v3.16b}, v28.16b
162
	tbl	v4.16b, {v4.16b}, v28.16b
163
	tbl	v5.16b, {v5.16b}, v28.16b
164
	tbl	v6.16b, {v6.16b}, v28.16b
165
	tbl	v7.16b, {v7.16b}, v28.16b
166
.Ldec_sbox:
167
	eor	v1.16b, v1.16b, v4.16b
168
	eor	v3.16b, v3.16b, v4.16b
169
	subs	x10, x10, #1
170
	eor	v4.16b, v4.16b, v7.16b
171
	eor	v2.16b, v2.16b, v7.16b
172
	eor	v1.16b, v1.16b, v6.16b
173
	eor	v6.16b, v6.16b, v4.16b
174
	eor	v2.16b, v2.16b, v5.16b
175
	eor	v0.16b, v0.16b, v1.16b
176
	eor	v7.16b, v7.16b, v6.16b
177
	eor	v8.16b, v6.16b, v2.16b
178
	and	v9.16b, v4.16b, v6.16b
179
	eor	v10.16b, v2.16b, v6.16b
180
	eor	v3.16b, v3.16b, v0.16b
181
	eor	v5.16b, v5.16b, v0.16b
182
	eor	v16.16b, v7.16b, v4.16b
183
	eor	v17.16b, v4.16b, v0.16b
184
	and	v18.16b, v0.16b, v2.16b
185
	eor	v19.16b, v7.16b, v4.16b
186
	eor	v1.16b, v1.16b, v3.16b
187
	eor	v20.16b, v3.16b, v0.16b
188
	eor	v21.16b, v5.16b, v2.16b
189
	eor	v22.16b, v3.16b, v7.16b
190
	and	v8.16b, v17.16b, v8.16b
191
	orr	v17.16b, v3.16b, v5.16b
192
	eor	v23.16b, v1.16b, v6.16b
193
	eor	v24.16b, v20.16b, v16.16b
194
	eor	v25.16b, v1.16b, v5.16b
195
	orr	v26.16b, v20.16b, v21.16b
196
	and	v20.16b, v20.16b, v21.16b
197
	and	v27.16b, v7.16b, v1.16b
198
	eor	v21.16b, v21.16b, v23.16b
199
	orr	v28.16b, v16.16b, v23.16b
200
	orr	v29.16b, v22.16b, v25.16b
201
	eor	v26.16b, v26.16b, v8.16b
202
	and	v16.16b, v16.16b, v23.16b
203
	and	v22.16b, v22.16b, v25.16b
204
	and	v21.16b, v24.16b, v21.16b
205
	eor	v8.16b, v28.16b, v8.16b
206
	eor	v23.16b, v5.16b, v2.16b
207
	eor	v24.16b, v1.16b, v6.16b
208
	eor	v16.16b, v16.16b, v22.16b
209
	eor	v22.16b, v3.16b, v0.16b
210
	eor	v25.16b, v29.16b, v21.16b
211
	eor	v21.16b, v26.16b, v21.16b
212
	eor	v8.16b, v8.16b, v20.16b
213
	eor	v26.16b, v23.16b, v24.16b
214
	eor	v16.16b, v16.16b, v20.16b
215
	eor	v28.16b, v22.16b, v19.16b
216
	eor	v20.16b, v25.16b, v20.16b
217
	eor	v9.16b, v21.16b, v9.16b
218
	eor	v8.16b, v8.16b, v18.16b
219
	eor	v18.16b, v5.16b, v1.16b
220
	eor	v21.16b, v16.16b, v17.16b
221
	eor	v16.16b, v16.16b, v17.16b
222
	eor	v17.16b, v20.16b, v27.16b
223
	eor	v20.16b, v3.16b, v7.16b
224
	eor	v25.16b, v9.16b, v8.16b
225
	eor	v27.16b, v0.16b, v4.16b
226
	and	v29.16b, v9.16b, v17.16b
227
	eor	v30.16b, v8.16b, v29.16b
228
	eor	v31.16b, v21.16b, v29.16b
229
	eor	v29.16b, v21.16b, v29.16b
230
	bsl	v30.16b, v17.16b, v21.16b
231
	bsl	v31.16b, v9.16b, v8.16b
232
	bsl	v16.16b, v30.16b, v29.16b
233
	bsl	v21.16b, v29.16b, v30.16b
234
	eor	v8.16b, v31.16b, v30.16b
235
	and	v1.16b, v1.16b, v31.16b
236
	and	v9.16b, v16.16b, v31.16b
237
	and	v6.16b, v6.16b, v30.16b
238
	eor	v16.16b, v17.16b, v21.16b
239
	and	v4.16b, v4.16b, v30.16b
240
	eor	v17.16b, v8.16b, v30.16b
241
	and	v21.16b, v24.16b, v8.16b
242
	eor	v9.16b, v9.16b, v25.16b
243
	and	v19.16b, v19.16b, v8.16b
244
	eor	v24.16b, v30.16b, v16.16b
245
	eor	v25.16b, v30.16b, v16.16b
246
	and	v7.16b, v7.16b, v17.16b
247
	and	v10.16b, v10.16b, v16.16b
248
	eor	v29.16b, v9.16b, v16.16b
249
	eor	v30.16b, v31.16b, v9.16b
250
	and	v0.16b, v24.16b, v0.16b
251
	and	v9.16b, v18.16b, v9.16b
252
	and	v2.16b, v25.16b, v2.16b
253
	eor	v10.16b, v10.16b, v6.16b
254
	eor	v18.16b, v29.16b, v16.16b
255
	and	v5.16b, v30.16b, v5.16b
256
	eor	v24.16b, v8.16b, v29.16b
257
	and	v25.16b, v26.16b, v29.16b
258
	and	v26.16b, v28.16b, v29.16b
259
	eor	v8.16b, v8.16b, v29.16b
260
	eor	v17.16b, v17.16b, v18.16b
261
	eor	v5.16b, v1.16b, v5.16b
262
	and	v23.16b, v24.16b, v23.16b
263
	eor	v21.16b, v21.16b, v25.16b
264
	eor	v19.16b, v19.16b, v26.16b
265
	eor	v0.16b, v4.16b, v0.16b
266
	and	v3.16b, v17.16b, v3.16b
267
	eor	v1.16b, v9.16b, v1.16b
268
	eor	v9.16b, v25.16b, v23.16b
269
	eor	v5.16b, v5.16b, v21.16b
270
	eor	v2.16b, v6.16b, v2.16b
271
	and	v6.16b, v8.16b, v22.16b
272
	eor	v3.16b, v7.16b, v3.16b
273
	and	v8.16b, v20.16b, v18.16b
274
	eor	v10.16b, v10.16b, v9.16b
275
	eor	v0.16b, v0.16b, v19.16b
276
	eor	v9.16b, v1.16b, v9.16b
277
	eor	v1.16b, v2.16b, v21.16b
278
	eor	v3.16b, v3.16b, v19.16b
279
	and	v16.16b, v27.16b, v16.16b
280
	eor	v17.16b, v26.16b, v6.16b
281
	eor	v6.16b, v8.16b, v7.16b
282
	eor	v7.16b, v1.16b, v9.16b
283
	eor	v1.16b, v5.16b, v3.16b
284
	eor	v2.16b, v10.16b, v3.16b
285
	eor	v4.16b, v16.16b, v4.16b
286
	eor	v8.16b, v6.16b, v17.16b
287
	eor	v5.16b, v9.16b, v3.16b
288
	eor	v9.16b, v0.16b, v1.16b
289
	eor	v6.16b, v7.16b, v1.16b
290
	eor	v0.16b, v4.16b, v17.16b
291
	eor	v4.16b, v8.16b, v7.16b
292
	eor	v7.16b, v9.16b, v2.16b
293
	eor	v8.16b, v3.16b, v0.16b
294
	eor	v7.16b, v7.16b, v5.16b
295
	eor	v3.16b, v4.16b, v7.16b
296
	eor	v4.16b, v7.16b, v0.16b
297
	eor	v7.16b, v8.16b, v3.16b
298
	bcc	.Ldec_done
299
	ext	v8.16b, v0.16b, v0.16b, #8
300
	ext	v9.16b, v1.16b, v1.16b, #8
301
	ldr	q28, [x11]                  // load from .LISR in common case (x10 > 0)
302
	ext	v10.16b, v6.16b, v6.16b, #8
303
	ext	v16.16b, v3.16b, v3.16b, #8
304
	ext	v17.16b, v5.16b, v5.16b, #8
305
	ext	v18.16b, v4.16b, v4.16b, #8
306
	eor	v8.16b, v8.16b, v0.16b
307
	eor	v9.16b, v9.16b, v1.16b
308
	eor	v10.16b, v10.16b, v6.16b
309
	eor	v16.16b, v16.16b, v3.16b
310
	eor	v17.16b, v17.16b, v5.16b
311
	ext	v19.16b, v2.16b, v2.16b, #8
312
	ext	v20.16b, v7.16b, v7.16b, #8
313
	eor	v18.16b, v18.16b, v4.16b
314
	eor	v6.16b, v6.16b, v8.16b
315
	eor	v8.16b, v2.16b, v10.16b
316
	eor	v4.16b, v4.16b, v9.16b
317
	eor	v2.16b, v19.16b, v2.16b
318
	eor	v9.16b, v20.16b, v7.16b
319
	eor	v0.16b, v0.16b, v16.16b
320
	eor	v1.16b, v1.16b, v16.16b
321
	eor	v6.16b, v6.16b, v17.16b
322
	eor	v8.16b, v8.16b, v16.16b
323
	eor	v7.16b, v7.16b, v18.16b
324
	eor	v4.16b, v4.16b, v16.16b
325
	eor	v2.16b, v3.16b, v2.16b
326
	eor	v1.16b, v1.16b, v17.16b
327
	eor	v3.16b, v5.16b, v9.16b
328
	eor	v5.16b, v8.16b, v17.16b
329
	eor	v7.16b, v7.16b, v17.16b
330
	ext	v8.16b, v0.16b, v0.16b, #12
331
	ext	v9.16b, v6.16b, v6.16b, #12
332
	ext	v10.16b, v4.16b, v4.16b, #12
333
	ext	v16.16b, v1.16b, v1.16b, #12
334
	ext	v17.16b, v5.16b, v5.16b, #12
335
	ext	v18.16b, v7.16b, v7.16b, #12
336
	eor	v0.16b, v0.16b, v8.16b
337
	eor	v6.16b, v6.16b, v9.16b
338
	eor	v4.16b, v4.16b, v10.16b
339
	ext	v19.16b, v2.16b, v2.16b, #12
340
	ext	v20.16b, v3.16b, v3.16b, #12
341
	eor	v1.16b, v1.16b, v16.16b
342
	eor	v5.16b, v5.16b, v17.16b
343
	eor	v7.16b, v7.16b, v18.16b
344
	eor	v2.16b, v2.16b, v19.16b
345
	eor	v16.16b, v16.16b, v0.16b
346
	eor	v3.16b, v3.16b, v20.16b
347
	eor	v17.16b, v17.16b, v4.16b
348
	eor	v10.16b, v10.16b, v6.16b
349
	ext	v0.16b, v0.16b, v0.16b, #8
350
	eor	v9.16b, v9.16b, v1.16b
351
	ext	v1.16b, v1.16b, v1.16b, #8
352
	eor	v8.16b, v8.16b, v3.16b
353
	eor	v16.16b, v16.16b, v3.16b
354
	eor	v18.16b, v18.16b, v5.16b
355
	eor	v19.16b, v19.16b, v7.16b
356
	ext	v21.16b, v5.16b, v5.16b, #8
357
	ext	v5.16b, v7.16b, v7.16b, #8
358
	eor	v7.16b, v20.16b, v2.16b
359
	ext	v4.16b, v4.16b, v4.16b, #8
360
	ext	v20.16b, v3.16b, v3.16b, #8
361
	eor	v17.16b, v17.16b, v3.16b
362
	ext	v2.16b, v2.16b, v2.16b, #8
363
	eor	v3.16b, v10.16b, v3.16b
364
	ext	v10.16b, v6.16b, v6.16b, #8
365
	eor	v0.16b, v0.16b, v8.16b
366
	eor	v1.16b, v1.16b, v16.16b
367
	eor	v5.16b, v5.16b, v18.16b
368
	eor	v3.16b, v3.16b, v4.16b
369
	eor	v7.16b, v20.16b, v7.16b
370
	eor	v6.16b, v2.16b, v19.16b
371
	eor	v4.16b, v21.16b, v17.16b
372
	eor	v2.16b, v10.16b, v9.16b
373
	bne	.Ldec_loop
374
	ldr	q28, [x11, #16]!            // load from .LISRM0 on last round (x10 == 0)
375
	b	.Ldec_loop
376
.align	4
377
.Ldec_done:
378
	ushr	v8.2d, v0.2d, #1
379
	movi	v9.16b, #0x55
380
	ldr	q10, [x9]
381
	ushr	v16.2d, v2.2d, #1
382
	movi	v17.16b, #0x33
383
	ushr	v18.2d, v6.2d, #1
384
	movi	v19.16b, #0x0f
385
	eor	v8.16b, v8.16b, v1.16b
386
	ushr	v20.2d, v3.2d, #1
387
	eor	v16.16b, v16.16b, v7.16b
388
	eor	v18.16b, v18.16b, v4.16b
389
	and	v8.16b, v8.16b, v9.16b
390
	eor	v20.16b, v20.16b, v5.16b
391
	and	v16.16b, v16.16b, v9.16b
392
	and	v18.16b, v18.16b, v9.16b
393
	shl	v21.2d, v8.2d, #1
394
	eor	v1.16b, v1.16b, v8.16b
395
	and	v8.16b, v20.16b, v9.16b
396
	eor	v7.16b, v7.16b, v16.16b
397
	shl	v9.2d, v16.2d, #1
398
	eor	v4.16b, v4.16b, v18.16b
399
	shl	v16.2d, v18.2d, #1
400
	eor	v0.16b, v0.16b, v21.16b
401
	shl	v18.2d, v8.2d, #1
402
	eor	v5.16b, v5.16b, v8.16b
403
	eor	v2.16b, v2.16b, v9.16b
404
	eor	v6.16b, v6.16b, v16.16b
405
	ushr	v8.2d, v1.2d, #2
406
	eor	v3.16b, v3.16b, v18.16b
407
	ushr	v9.2d, v0.2d, #2
408
	ushr	v16.2d, v7.2d, #2
409
	ushr	v18.2d, v2.2d, #2
410
	eor	v8.16b, v8.16b, v4.16b
411
	eor	v9.16b, v9.16b, v6.16b
412
	eor	v16.16b, v16.16b, v5.16b
413
	eor	v18.16b, v18.16b, v3.16b
414
	and	v8.16b, v8.16b, v17.16b
415
	and	v9.16b, v9.16b, v17.16b
416
	and	v16.16b, v16.16b, v17.16b
417
	and	v17.16b, v18.16b, v17.16b
418
	eor	v4.16b, v4.16b, v8.16b
419
	shl	v8.2d, v8.2d, #2
420
	eor	v6.16b, v6.16b, v9.16b
421
	shl	v9.2d, v9.2d, #2
422
	eor	v5.16b, v5.16b, v16.16b
423
	shl	v16.2d, v16.2d, #2
424
	eor	v3.16b, v3.16b, v17.16b
425
	shl	v17.2d, v17.2d, #2
426
	eor	v1.16b, v1.16b, v8.16b
427
	eor	v0.16b, v0.16b, v9.16b
428
	eor	v7.16b, v7.16b, v16.16b
429
	eor	v2.16b, v2.16b, v17.16b
430
	ushr	v8.2d, v4.2d, #4
431
	ushr	v9.2d, v6.2d, #4
432
	ushr	v16.2d, v1.2d, #4
433
	ushr	v17.2d, v0.2d, #4
434
	eor	v8.16b, v8.16b, v5.16b
435
	eor	v9.16b, v9.16b, v3.16b
436
	eor	v16.16b, v16.16b, v7.16b
437
	eor	v17.16b, v17.16b, v2.16b
438
	and	v8.16b, v8.16b, v19.16b
439
	and	v9.16b, v9.16b, v19.16b
440
	and	v16.16b, v16.16b, v19.16b
441
	and	v17.16b, v17.16b, v19.16b
442
	eor	v5.16b, v5.16b, v8.16b
443
	shl	v8.2d, v8.2d, #4
444
	eor	v3.16b, v3.16b, v9.16b
445
	shl	v9.2d, v9.2d, #4
446
	eor	v7.16b, v7.16b, v16.16b
447
	shl	v16.2d, v16.2d, #4
448
	eor	v2.16b, v2.16b, v17.16b
449
	shl	v17.2d, v17.2d, #4
450
	eor	v4.16b, v4.16b, v8.16b
451
	eor	v6.16b, v6.16b, v9.16b
452
	eor	v7.16b, v7.16b, v10.16b
453
	eor	v1.16b, v1.16b, v16.16b
454
	eor	v2.16b, v2.16b, v10.16b
455
	eor	v0.16b, v0.16b, v17.16b
456
	eor	v4.16b, v4.16b, v10.16b
457
	eor	v6.16b, v6.16b, v10.16b
458
	eor	v3.16b, v3.16b, v10.16b
459
	eor	v5.16b, v5.16b, v10.16b
460
	eor	v1.16b, v1.16b, v10.16b
461
	eor	v0.16b, v0.16b, v10.16b
462
	ret
463
.size	_bsaes_decrypt8,.-_bsaes_decrypt8
464

465
.section	.rodata
466
.type	_bsaes_consts,%object
467
.align	6
468
_bsaes_consts:
469
// InvShiftRows constants
470
// Used in _bsaes_decrypt8, which assumes contiguity
471
// .LM0ISR used with round 0 key
472
// .LISR   used with middle round keys
473
// .LISRM0 used with final round key
474
.LM0ISR:
475
.quad	0x0a0e0206070b0f03, 0x0004080c0d010509
476
.LISR:
477
.quad	0x0504070602010003, 0x0f0e0d0c080b0a09
478
.LISRM0:
479
.quad	0x01040b0e0205080f, 0x0306090c00070a0d
480

481
// ShiftRows constants
482
// Used in _bsaes_encrypt8, which assumes contiguity
483
// .LM0SR used with round 0 key
484
// .LSR   used with middle round keys
485
// .LSRM0 used with final round key
486
.LM0SR:
487
.quad	0x0a0e02060f03070b, 0x0004080c05090d01
488
.LSR:
489
.quad	0x0504070600030201, 0x0f0e0d0c0a09080b
490
.LSRM0:
491
.quad	0x0304090e00050a0f, 0x01060b0c0207080d
492

493
.LM0_bigendian:
494
.quad	0x02060a0e03070b0f, 0x0004080c0105090d
495
.LM0_littleendian:
496
.quad	0x0105090d0004080c, 0x03070b0f02060a0e
497

498
// Used in ossl_bsaes_ctr32_encrypt_blocks, prior to dropping into
499
// _bsaes_encrypt8_alt, for round 0 key in place of .LM0SR
500
.LREVM0SR:
501
.quad	0x090d01050c000408, 0x03070b0f060a0e02
502

503
.align	6
504
.size	_bsaes_consts,.-_bsaes_consts
505

506
.previous
507

508
.type	_bsaes_encrypt8,%function
509
.align	4
510
// On entry:
511
//   x9 -> key (previously expanded using _bsaes_key_convert)
512
//   x10 = number of rounds
513
//   v0-v7 input data
514
// On exit:
515
//   x9-x11 corrupted
516
//   other general-purpose registers preserved
517
//   v0-v7 output data
518
//   v11-v15 preserved
519
//   other SIMD registers corrupted
520
_bsaes_encrypt8:
521
	ldr	q8, [x9], #16
522
	adrp	x11, .LM0SR
523
	add	x11, x11, #:lo12:.LM0SR
524
	ldr	q9, [x11], #16
525
_bsaes_encrypt8_alt:
526
	eor	v0.16b, v0.16b, v8.16b
527
	eor	v1.16b, v1.16b, v8.16b
528
	sub	x10, x10, #1
529
	eor	v2.16b, v2.16b, v8.16b
530
	eor	v4.16b, v4.16b, v8.16b
531
	eor	v3.16b, v3.16b, v8.16b
532
	eor	v5.16b, v5.16b, v8.16b
533
	tbl	v0.16b, {v0.16b}, v9.16b
534
	tbl	v1.16b, {v1.16b}, v9.16b
535
	tbl	v2.16b, {v2.16b}, v9.16b
536
	tbl	v4.16b, {v4.16b}, v9.16b
537
	eor	v6.16b, v6.16b, v8.16b
538
	eor	v7.16b, v7.16b, v8.16b
539
	tbl	v3.16b, {v3.16b}, v9.16b
540
	tbl	v5.16b, {v5.16b}, v9.16b
541
	tbl	v6.16b, {v6.16b}, v9.16b
542
	ushr	v8.2d, v0.2d, #1
543
	movi	v10.16b, #0x55
544
	tbl	v7.16b, {v7.16b}, v9.16b
545
	ushr	v9.2d, v4.2d, #1
546
	movi	v16.16b, #0x33
547
	ushr	v17.2d, v2.2d, #1
548
	eor	v8.16b, v8.16b, v1.16b
549
	movi	v18.16b, #0x0f
550
	ushr	v19.2d, v6.2d, #1
551
	eor	v9.16b, v9.16b, v5.16b
552
	eor	v17.16b, v17.16b, v3.16b
553
	and	v8.16b, v8.16b, v10.16b
554
	eor	v19.16b, v19.16b, v7.16b
555
	and	v9.16b, v9.16b, v10.16b
556
	and	v17.16b, v17.16b, v10.16b
557
	eor	v1.16b, v1.16b, v8.16b
558
	shl	v8.2d, v8.2d, #1
559
	and	v10.16b, v19.16b, v10.16b
560
	eor	v5.16b, v5.16b, v9.16b
561
	shl	v9.2d, v9.2d, #1
562
	eor	v3.16b, v3.16b, v17.16b
563
	shl	v17.2d, v17.2d, #1
564
	eor	v0.16b, v0.16b, v8.16b
565
	shl	v8.2d, v10.2d, #1
566
	eor	v7.16b, v7.16b, v10.16b
567
	eor	v4.16b, v4.16b, v9.16b
568
	eor	v2.16b, v2.16b, v17.16b
569
	ushr	v9.2d, v1.2d, #2
570
	eor	v6.16b, v6.16b, v8.16b
571
	ushr	v8.2d, v0.2d, #2
572
	ushr	v10.2d, v5.2d, #2
573
	ushr	v17.2d, v4.2d, #2
574
	eor	v9.16b, v9.16b, v3.16b
575
	eor	v8.16b, v8.16b, v2.16b
576
	eor	v10.16b, v10.16b, v7.16b
577
	eor	v17.16b, v17.16b, v6.16b
578
	and	v9.16b, v9.16b, v16.16b
579
	and	v8.16b, v8.16b, v16.16b
580
	and	v10.16b, v10.16b, v16.16b
581
	and	v16.16b, v17.16b, v16.16b
582
	eor	v3.16b, v3.16b, v9.16b
583
	shl	v9.2d, v9.2d, #2
584
	eor	v2.16b, v2.16b, v8.16b
585
	shl	v8.2d, v8.2d, #2
586
	eor	v7.16b, v7.16b, v10.16b
587
	shl	v10.2d, v10.2d, #2
588
	eor	v6.16b, v6.16b, v16.16b
589
	shl	v16.2d, v16.2d, #2
590
	eor	v1.16b, v1.16b, v9.16b
591
	eor	v0.16b, v0.16b, v8.16b
592
	eor	v5.16b, v5.16b, v10.16b
593
	eor	v4.16b, v4.16b, v16.16b
594
	ushr	v8.2d, v3.2d, #4
595
	ushr	v9.2d, v2.2d, #4
596
	ushr	v10.2d, v1.2d, #4
597
	ushr	v16.2d, v0.2d, #4
598
	eor	v8.16b, v8.16b, v7.16b
599
	eor	v9.16b, v9.16b, v6.16b
600
	eor	v10.16b, v10.16b, v5.16b
601
	eor	v16.16b, v16.16b, v4.16b
602
	and	v8.16b, v8.16b, v18.16b
603
	and	v9.16b, v9.16b, v18.16b
604
	and	v10.16b, v10.16b, v18.16b
605
	and	v16.16b, v16.16b, v18.16b
606
	eor	v7.16b, v7.16b, v8.16b
607
	shl	v8.2d, v8.2d, #4
608
	eor	v6.16b, v6.16b, v9.16b
609
	shl	v9.2d, v9.2d, #4
610
	eor	v5.16b, v5.16b, v10.16b
611
	shl	v10.2d, v10.2d, #4
612
	eor	v4.16b, v4.16b, v16.16b
613
	shl	v16.2d, v16.2d, #4
614
	eor	v3.16b, v3.16b, v8.16b
615
	eor	v2.16b, v2.16b, v9.16b
616
	eor	v1.16b, v1.16b, v10.16b
617
	eor	v0.16b, v0.16b, v16.16b
618
	b	.Lenc_sbox
619
.align	4
620
.Lenc_loop:
621
	ld1	{v16.16b, v17.16b, v18.16b, v19.16b}, [x9], #64
622
	ldp	q8, q9, [x9], #32
623
	eor	v0.16b, v16.16b, v0.16b
624
	ldr	q10, [x9], #16
625
	eor	v1.16b, v17.16b, v1.16b
626
	ldr	q16, [x9], #16
627
	eor	v2.16b, v18.16b, v2.16b
628
	eor	v3.16b, v19.16b, v3.16b
629
	eor	v4.16b, v8.16b, v4.16b
630
	eor	v5.16b, v9.16b, v5.16b
631
	eor	v6.16b, v10.16b, v6.16b
632
	eor	v7.16b, v16.16b, v7.16b
633
	tbl	v0.16b, {v0.16b}, v28.16b
634
	tbl	v1.16b, {v1.16b}, v28.16b
635
	tbl	v2.16b, {v2.16b}, v28.16b
636
	tbl	v3.16b, {v3.16b}, v28.16b
637
	tbl	v4.16b, {v4.16b}, v28.16b
638
	tbl	v5.16b, {v5.16b}, v28.16b
639
	tbl	v6.16b, {v6.16b}, v28.16b
640
	tbl	v7.16b, {v7.16b}, v28.16b
641
.Lenc_sbox:
642
	eor	v5.16b, v5.16b, v6.16b
643
	eor	v3.16b, v3.16b, v0.16b
644
	subs	x10, x10, #1
645
	eor	v2.16b, v2.16b, v1.16b
646
	eor	v5.16b, v5.16b, v0.16b
647
	eor	v8.16b, v3.16b, v7.16b
648
	eor	v6.16b, v6.16b, v2.16b
649
	eor	v7.16b, v7.16b, v5.16b
650
	eor	v8.16b, v8.16b, v4.16b
651
	eor	v3.16b, v6.16b, v3.16b
652
	eor	v4.16b, v4.16b, v5.16b
653
	eor	v6.16b, v1.16b, v5.16b
654
	eor	v2.16b, v2.16b, v7.16b
655
	eor	v1.16b, v8.16b, v1.16b
656
	eor	v8.16b, v7.16b, v4.16b
657
	eor	v9.16b, v3.16b, v0.16b
658
	eor	v10.16b, v7.16b, v6.16b
659
	eor	v16.16b, v5.16b, v3.16b
660
	eor	v17.16b, v6.16b, v2.16b
661
	eor	v18.16b, v5.16b, v1.16b
662
	eor	v19.16b, v2.16b, v4.16b
663
	eor	v20.16b, v1.16b, v0.16b
664
	orr	v21.16b, v8.16b, v9.16b
665
	orr	v22.16b, v10.16b, v16.16b
666
	eor	v23.16b, v8.16b, v17.16b
667
	eor	v24.16b, v9.16b, v18.16b
668
	and	v19.16b, v19.16b, v20.16b
669
	orr	v20.16b, v17.16b, v18.16b
670
	and	v8.16b, v8.16b, v9.16b
671
	and	v9.16b, v17.16b, v18.16b
672
	and	v17.16b, v23.16b, v24.16b
673
	and	v10.16b, v10.16b, v16.16b
674
	eor	v16.16b, v21.16b, v19.16b
675
	eor	v18.16b, v20.16b, v19.16b
676
	and	v19.16b, v2.16b, v1.16b
677
	and	v20.16b, v6.16b, v5.16b
678
	eor	v21.16b, v22.16b, v17.16b
679
	eor	v9.16b, v9.16b, v10.16b
680
	eor	v10.16b, v16.16b, v17.16b
681
	eor	v16.16b, v18.16b, v8.16b
682
	and	v17.16b, v4.16b, v0.16b
683
	orr	v18.16b, v7.16b, v3.16b
684
	eor	v21.16b, v21.16b, v8.16b
685
	eor	v8.16b, v9.16b, v8.16b
686
	eor	v9.16b, v10.16b, v19.16b
687
	eor	v10.16b, v3.16b, v0.16b
688
	eor	v16.16b, v16.16b, v17.16b
689
	eor	v17.16b, v5.16b, v1.16b
690
	eor	v19.16b, v21.16b, v20.16b
691
	eor	v20.16b, v8.16b, v18.16b
692
	eor	v8.16b, v8.16b, v18.16b
693
	eor	v18.16b, v7.16b, v4.16b
694
	eor	v21.16b, v9.16b, v16.16b
695
	eor	v22.16b, v6.16b, v2.16b
696
	and	v23.16b, v9.16b, v19.16b
697
	eor	v24.16b, v10.16b, v17.16b
698
	eor	v25.16b, v0.16b, v1.16b
699
	eor	v26.16b, v7.16b, v6.16b
700
	eor	v27.16b, v18.16b, v22.16b
701
	eor	v28.16b, v3.16b, v5.16b
702
	eor	v29.16b, v16.16b, v23.16b
703
	eor	v30.16b, v20.16b, v23.16b
704
	eor	v23.16b, v20.16b, v23.16b
705
	eor	v31.16b, v4.16b, v2.16b
706
	bsl	v29.16b, v19.16b, v20.16b
707
	bsl	v30.16b, v9.16b, v16.16b
708
	bsl	v8.16b, v29.16b, v23.16b
709
	bsl	v20.16b, v23.16b, v29.16b
710
	eor	v9.16b, v30.16b, v29.16b
711
	and	v5.16b, v5.16b, v30.16b
712
	and	v8.16b, v8.16b, v30.16b
713
	and	v1.16b, v1.16b, v29.16b
714
	eor	v16.16b, v19.16b, v20.16b
715
	and	v2.16b, v2.16b, v29.16b
716
	eor	v19.16b, v9.16b, v29.16b
717
	and	v17.16b, v17.16b, v9.16b
718
	eor	v8.16b, v8.16b, v21.16b
719
	and	v20.16b, v22.16b, v9.16b
720
	eor	v21.16b, v29.16b, v16.16b
721
	eor	v22.16b, v29.16b, v16.16b
722
	and	v23.16b, v25.16b, v16.16b
723
	and	v6.16b, v6.16b, v19.16b
724
	eor	v25.16b, v8.16b, v16.16b
725
	eor	v29.16b, v30.16b, v8.16b
726
	and	v4.16b, v21.16b, v4.16b
727
	and	v8.16b, v28.16b, v8.16b
728
	and	v0.16b, v22.16b, v0.16b
729
	eor	v21.16b, v23.16b, v1.16b
730
	eor	v22.16b, v9.16b, v25.16b
731
	eor	v9.16b, v9.16b, v25.16b
732
	eor	v23.16b, v25.16b, v16.16b
733
	and	v3.16b, v29.16b, v3.16b
734
	and	v24.16b, v24.16b, v25.16b
735
	and	v25.16b, v27.16b, v25.16b
736
	and	v10.16b, v22.16b, v10.16b
737
	and	v9.16b, v9.16b, v18.16b
738
	eor	v18.16b, v19.16b, v23.16b
739
	and	v19.16b, v26.16b, v23.16b
740
	eor	v3.16b, v5.16b, v3.16b
741
	eor	v17.16b, v17.16b, v24.16b
742
	eor	v10.16b, v24.16b, v10.16b
743
	and	v16.16b, v31.16b, v16.16b
744
	eor	v20.16b, v20.16b, v25.16b
745
	eor	v9.16b, v25.16b, v9.16b
746
	eor	v4.16b, v2.16b, v4.16b
747
	and	v7.16b, v18.16b, v7.16b
748
	eor	v18.16b, v19.16b, v6.16b
749
	eor	v5.16b, v8.16b, v5.16b
750
	eor	v0.16b, v1.16b, v0.16b
751
	eor	v1.16b, v21.16b, v10.16b
752
	eor	v8.16b, v3.16b, v17.16b
753
	eor	v2.16b, v16.16b, v2.16b
754
	eor	v3.16b, v6.16b, v7.16b
755
	eor	v6.16b, v18.16b, v9.16b
756
	eor	v4.16b, v4.16b, v20.16b
757
	eor	v10.16b, v5.16b, v10.16b
758
	eor	v0.16b, v0.16b, v17.16b
759
	eor	v9.16b, v2.16b, v9.16b
760
	eor	v3.16b, v3.16b, v20.16b
761
	eor	v7.16b, v6.16b, v1.16b
762
	eor	v5.16b, v8.16b, v4.16b
763
	eor	v6.16b, v10.16b, v1.16b
764
	eor	v2.16b, v4.16b, v0.16b
765
	eor	v4.16b, v3.16b, v10.16b
766
	eor	v9.16b, v9.16b, v7.16b
767
	eor	v3.16b, v0.16b, v5.16b
768
	eor	v0.16b, v1.16b, v4.16b
769
	eor	v1.16b, v4.16b, v8.16b
770
	eor	v4.16b, v9.16b, v5.16b
771
	eor	v6.16b, v6.16b, v3.16b
772
	bcc	.Lenc_done
773
	ext	v8.16b, v0.16b, v0.16b, #12
774
	ext	v9.16b, v4.16b, v4.16b, #12
775
	ldr	q28, [x11]
776
	ext	v10.16b, v6.16b, v6.16b, #12
777
	ext	v16.16b, v1.16b, v1.16b, #12
778
	ext	v17.16b, v3.16b, v3.16b, #12
779
	ext	v18.16b, v7.16b, v7.16b, #12
780
	eor	v0.16b, v0.16b, v8.16b
781
	eor	v4.16b, v4.16b, v9.16b
782
	eor	v6.16b, v6.16b, v10.16b
783
	ext	v19.16b, v2.16b, v2.16b, #12
784
	ext	v20.16b, v5.16b, v5.16b, #12
785
	eor	v1.16b, v1.16b, v16.16b
786
	eor	v3.16b, v3.16b, v17.16b
787
	eor	v7.16b, v7.16b, v18.16b
788
	eor	v2.16b, v2.16b, v19.16b
789
	eor	v16.16b, v16.16b, v0.16b
790
	eor	v5.16b, v5.16b, v20.16b
791
	eor	v17.16b, v17.16b, v6.16b
792
	eor	v10.16b, v10.16b, v4.16b
793
	ext	v0.16b, v0.16b, v0.16b, #8
794
	eor	v9.16b, v9.16b, v1.16b
795
	ext	v1.16b, v1.16b, v1.16b, #8
796
	eor	v8.16b, v8.16b, v5.16b
797
	eor	v16.16b, v16.16b, v5.16b
798
	eor	v18.16b, v18.16b, v3.16b
799
	eor	v19.16b, v19.16b, v7.16b
800
	ext	v3.16b, v3.16b, v3.16b, #8
801
	ext	v7.16b, v7.16b, v7.16b, #8
802
	eor	v20.16b, v20.16b, v2.16b
803
	ext	v6.16b, v6.16b, v6.16b, #8
804
	ext	v21.16b, v5.16b, v5.16b, #8
805
	eor	v17.16b, v17.16b, v5.16b
806
	ext	v2.16b, v2.16b, v2.16b, #8
807
	eor	v10.16b, v10.16b, v5.16b
808
	ext	v22.16b, v4.16b, v4.16b, #8
809
	eor	v0.16b, v0.16b, v8.16b
810
	eor	v1.16b, v1.16b, v16.16b
811
	eor	v5.16b, v7.16b, v18.16b
812
	eor	v4.16b, v3.16b, v17.16b
813
	eor	v3.16b, v6.16b, v10.16b
814
	eor	v7.16b, v21.16b, v20.16b
815
	eor	v6.16b, v2.16b, v19.16b
816
	eor	v2.16b, v22.16b, v9.16b
817
	bne	.Lenc_loop
818
	ldr	q28, [x11, #16]!            // load from .LSRM0 on last round (x10 == 0)
819
	b	.Lenc_loop
820
.align	4
821
.Lenc_done:
822
	ushr	v8.2d, v0.2d, #1
823
	movi	v9.16b, #0x55
824
	ldr	q10, [x9]
825
	ushr	v16.2d, v3.2d, #1
826
	movi	v17.16b, #0x33
827
	ushr	v18.2d, v4.2d, #1
828
	movi	v19.16b, #0x0f
829
	eor	v8.16b, v8.16b, v1.16b
830
	ushr	v20.2d, v2.2d, #1
831
	eor	v16.16b, v16.16b, v7.16b
832
	eor	v18.16b, v18.16b, v6.16b
833
	and	v8.16b, v8.16b, v9.16b
834
	eor	v20.16b, v20.16b, v5.16b
835
	and	v16.16b, v16.16b, v9.16b
836
	and	v18.16b, v18.16b, v9.16b
837
	shl	v21.2d, v8.2d, #1
838
	eor	v1.16b, v1.16b, v8.16b
839
	and	v8.16b, v20.16b, v9.16b
840
	eor	v7.16b, v7.16b, v16.16b
841
	shl	v9.2d, v16.2d, #1
842
	eor	v6.16b, v6.16b, v18.16b
843
	shl	v16.2d, v18.2d, #1
844
	eor	v0.16b, v0.16b, v21.16b
845
	shl	v18.2d, v8.2d, #1
846
	eor	v5.16b, v5.16b, v8.16b
847
	eor	v3.16b, v3.16b, v9.16b
848
	eor	v4.16b, v4.16b, v16.16b
849
	ushr	v8.2d, v1.2d, #2
850
	eor	v2.16b, v2.16b, v18.16b
851
	ushr	v9.2d, v0.2d, #2
852
	ushr	v16.2d, v7.2d, #2
853
	ushr	v18.2d, v3.2d, #2
854
	eor	v8.16b, v8.16b, v6.16b
855
	eor	v9.16b, v9.16b, v4.16b
856
	eor	v16.16b, v16.16b, v5.16b
857
	eor	v18.16b, v18.16b, v2.16b
858
	and	v8.16b, v8.16b, v17.16b
859
	and	v9.16b, v9.16b, v17.16b
860
	and	v16.16b, v16.16b, v17.16b
861
	and	v17.16b, v18.16b, v17.16b
862
	eor	v6.16b, v6.16b, v8.16b
863
	shl	v8.2d, v8.2d, #2
864
	eor	v4.16b, v4.16b, v9.16b
865
	shl	v9.2d, v9.2d, #2
866
	eor	v5.16b, v5.16b, v16.16b
867
	shl	v16.2d, v16.2d, #2
868
	eor	v2.16b, v2.16b, v17.16b
869
	shl	v17.2d, v17.2d, #2
870
	eor	v1.16b, v1.16b, v8.16b
871
	eor	v0.16b, v0.16b, v9.16b
872
	eor	v7.16b, v7.16b, v16.16b
873
	eor	v3.16b, v3.16b, v17.16b
874
	ushr	v8.2d, v6.2d, #4
875
	ushr	v9.2d, v4.2d, #4
876
	ushr	v16.2d, v1.2d, #4
877
	ushr	v17.2d, v0.2d, #4
878
	eor	v8.16b, v8.16b, v5.16b
879
	eor	v9.16b, v9.16b, v2.16b
880
	eor	v16.16b, v16.16b, v7.16b
881
	eor	v17.16b, v17.16b, v3.16b
882
	and	v8.16b, v8.16b, v19.16b
883
	and	v9.16b, v9.16b, v19.16b
884
	and	v16.16b, v16.16b, v19.16b
885
	and	v17.16b, v17.16b, v19.16b
886
	eor	v5.16b, v5.16b, v8.16b
887
	shl	v8.2d, v8.2d, #4
888
	eor	v2.16b, v2.16b, v9.16b
889
	shl	v9.2d, v9.2d, #4
890
	eor	v7.16b, v7.16b, v16.16b
891
	shl	v16.2d, v16.2d, #4
892
	eor	v3.16b, v3.16b, v17.16b
893
	shl	v17.2d, v17.2d, #4
894
	eor	v6.16b, v6.16b, v8.16b
895
	eor	v4.16b, v4.16b, v9.16b
896
	eor	v7.16b, v7.16b, v10.16b
897
	eor	v1.16b, v1.16b, v16.16b
898
	eor	v3.16b, v3.16b, v10.16b
899
	eor	v0.16b, v0.16b, v17.16b
900
	eor	v6.16b, v6.16b, v10.16b
901
	eor	v4.16b, v4.16b, v10.16b
902
	eor	v2.16b, v2.16b, v10.16b
903
	eor	v5.16b, v5.16b, v10.16b
904
	eor	v1.16b, v1.16b, v10.16b
905
	eor	v0.16b, v0.16b, v10.16b
906
	ret
907
.size	_bsaes_encrypt8,.-_bsaes_encrypt8
908

909
.type	_bsaes_key_convert,%function
910
.align	4
911
// On entry:
912
//   x9 -> input key (big-endian)
913
//   x10 = number of rounds
914
//   x17 -> output key (native endianness)
915
// On exit:
916
//   x9, x10 corrupted
917
//   x11 -> .LM0_bigendian
918
//   x17 -> last quadword of output key
919
//   other general-purpose registers preserved
920
//   v2-v6 preserved
921
//   v7.16b[] = 0x63
922
//   v8-v14 preserved
923
//   v15 = last round key (converted to native endianness)
924
//   other SIMD registers corrupted
925
_bsaes_key_convert:
926
#ifdef __AARCH64EL__
927
	adrp	x11, .LM0_littleendian
928
	add	x11, x11, #:lo12:.LM0_littleendian
929
#else
930
	adrp	x11, .LM0_bigendian
931
	add	x11, x11, #:lo12:.LM0_bigendian
932
#endif
933
	ldr	q0, [x9], #16               // load round 0 key
934
	ldr	q1, [x11]                   // .LM0
935
	ldr	q15, [x9], #16              // load round 1 key
936

937
	movi	v7.16b, #0x63               // compose .L63
938
	movi	v16.16b, #0x01              // bit masks
939
	movi	v17.16b, #0x02
940
	movi	v18.16b, #0x04
941
	movi	v19.16b, #0x08
942
	movi	v20.16b, #0x10
943
	movi	v21.16b, #0x20
944
	movi	v22.16b, #0x40
945
	movi	v23.16b, #0x80
946

947
#ifdef __AARCH64EL__
948
	rev32	v0.16b, v0.16b
949
#endif
950
	sub	x10, x10, #1
951
	str	q0, [x17], #16              // save round 0 key
952

953
.align	4
954
.Lkey_loop:
955
	tbl	v0.16b, {v15.16b}, v1.16b
956
	ldr	q15, [x9], #16              // load next round key
957

958
	eor	v0.16b, v0.16b, v7.16b
959
	cmtst	v24.16b, v0.16b, v16.16b
960
	cmtst	v25.16b, v0.16b, v17.16b
961
	cmtst	v26.16b, v0.16b, v18.16b
962
	cmtst	v27.16b, v0.16b, v19.16b
963
	cmtst	v28.16b, v0.16b, v20.16b
964
	cmtst	v29.16b, v0.16b, v21.16b
965
	cmtst	v30.16b, v0.16b, v22.16b
966
	cmtst	v31.16b, v0.16b, v23.16b
967
	sub	x10, x10, #1
968
	st1	{v24.16b,v25.16b,v26.16b,v27.16b}, [x17], #64 // write bit-sliced round key
969
	st1	{v28.16b,v29.16b,v30.16b,v31.16b}, [x17], #64
970
	cbnz	x10, .Lkey_loop
971

972
        // don't save last round key
973
#ifdef __AARCH64EL__
974
	rev32	v15.16b, v15.16b
975
	adrp	x11, .LM0_bigendian
976
	add	x11, x11, #:lo12:.LM0_bigendian
977
#endif
978
	ret
979
.size	_bsaes_key_convert,.-_bsaes_key_convert
980

981
.globl	ossl_bsaes_cbc_encrypt
982
.type	ossl_bsaes_cbc_encrypt,%function
983
.align	4
984
// On entry:
985
//   x0 -> input ciphertext
986
//   x1 -> output plaintext
987
//   x2 = size of ciphertext and plaintext in bytes (assumed a multiple of 16)
988
//   x3 -> key
989
//   x4 -> 128-bit initialisation vector (or preceding 128-bit block of ciphertext if continuing after an earlier call)
990
//   w5 must be == 0
991
// On exit:
992
//   Output plaintext filled in
993
//   Initialisation vector overwritten with last quadword of ciphertext
994
//   No output registers, usual AAPCS64 register preservation
995
ossl_bsaes_cbc_encrypt:
996
	AARCH64_VALID_CALL_TARGET
997
	cmp	x2, #128
998
	bhs	.Lcbc_do_bsaes
999
	b	AES_cbc_encrypt
1000
.Lcbc_do_bsaes:
1001

1002
        // it is up to the caller to make sure we are called with enc == 0
1003

1004
	stp	x29, x30, [sp, #-48]!
1005
	stp	d8, d9, [sp, #16]
1006
	stp	d10, d15, [sp, #32]
1007
	lsr	x2, x2, #4                  // len in 16 byte blocks
1008

1009
	ldr	w15, [x3, #240]             // get # of rounds
1010
	mov	x14, sp
1011

1012
        // allocate the key schedule on the stack
1013
	add	x17, sp, #96
1014
	sub	x17, x17, x15, lsl #7       // 128 bytes per inner round key, less 96 bytes
1015

1016
        // populate the key schedule
1017
	mov	x9, x3                      // pass key
1018
	mov	x10, x15                    // pass # of rounds
1019
	mov	sp, x17                     // sp is sp
1020
	bl	_bsaes_key_convert
1021
	ldr	q6,  [sp]
1022
	str	q15, [x17]                  // save last round key
1023
	eor	v6.16b, v6.16b, v7.16b      // fix up round 0 key (by XORing with 0x63)
1024
	str	q6, [sp]
1025

1026
	ldr	q15, [x4]                   // load IV
1027
	b	.Lcbc_dec_loop
1028

1029
.align	4
1030
.Lcbc_dec_loop:
1031
	subs	x2, x2, #0x8
1032
	bmi	.Lcbc_dec_loop_finish
1033

1034
	ldr	q0, [x0], #16               // load input
1035
	mov	x9, sp                      // pass the key
1036
	ldr	q1, [x0], #16
1037
	mov	x10, x15
1038
	ldr	q2, [x0], #16
1039
	ldr	q3, [x0], #16
1040
	ldr	q4, [x0], #16
1041
	ldr	q5, [x0], #16
1042
	ldr	q6, [x0], #16
1043
	ldr	q7, [x0], #-7*16
1044

1045
	bl	_bsaes_decrypt8
1046

1047
	ldr	q16, [x0], #16              // reload input
1048
	eor	v0.16b, v0.16b, v15.16b     // ^= IV
1049
	eor	v1.16b, v1.16b, v16.16b
1050
	str	q0, [x1], #16               // write output
1051
	ldr	q0, [x0], #16
1052
	str	q1, [x1], #16
1053
	ldr	q1, [x0], #16
1054
	eor	v1.16b, v4.16b, v1.16b
1055
	ldr	q4, [x0], #16
1056
	eor	v2.16b, v2.16b, v4.16b
1057
	eor	v0.16b, v6.16b, v0.16b
1058
	ldr	q4, [x0], #16
1059
	str	q0, [x1], #16
1060
	str	q1, [x1], #16
1061
	eor	v0.16b, v7.16b, v4.16b
1062
	ldr	q1, [x0], #16
1063
	str	q2, [x1], #16
1064
	ldr	q2, [x0], #16
1065
	ldr	q15, [x0], #16
1066
	str	q0, [x1], #16
1067
	eor	v0.16b, v5.16b, v2.16b
1068
	eor	v1.16b, v3.16b, v1.16b
1069
	str	q1, [x1], #16
1070
	str	q0, [x1], #16
1071

1072
	b	.Lcbc_dec_loop
1073

1074
.Lcbc_dec_loop_finish:
1075
	adds	x2, x2, #8
1076
	beq	.Lcbc_dec_done
1077

1078
	ldr	q0, [x0], #16               // load input
1079
	cmp	x2, #2
1080
	blo	.Lcbc_dec_one
1081
	ldr	q1, [x0], #16
1082
	mov	x9, sp                      // pass the key
1083
	mov	x10, x15
1084
	beq	.Lcbc_dec_two
1085
	ldr	q2, [x0], #16
1086
	cmp	x2, #4
1087
	blo	.Lcbc_dec_three
1088
	ldr	q3, [x0], #16
1089
	beq	.Lcbc_dec_four
1090
	ldr	q4, [x0], #16
1091
	cmp	x2, #6
1092
	blo	.Lcbc_dec_five
1093
	ldr	q5, [x0], #16
1094
	beq	.Lcbc_dec_six
1095
	ldr	q6, [x0], #-6*16
1096

1097
	bl	_bsaes_decrypt8
1098

1099
	ldr	q5, [x0], #16               // reload input
1100
	eor	v0.16b, v0.16b, v15.16b     // ^= IV
1101
	ldr	q8, [x0], #16
1102
	ldr	q9, [x0], #16
1103
	ldr	q10, [x0], #16
1104
	str	q0, [x1], #16               // write output
1105
	ldr	q0, [x0], #16
1106
	eor	v1.16b, v1.16b, v5.16b
1107
	ldr	q5, [x0], #16
1108
	eor	v6.16b, v6.16b, v8.16b
1109
	ldr	q15, [x0]
1110
	eor	v4.16b, v4.16b, v9.16b
1111
	eor	v2.16b, v2.16b, v10.16b
1112
	str	q1, [x1], #16
1113
	eor	v0.16b, v7.16b, v0.16b
1114
	str	q6, [x1], #16
1115
	eor	v1.16b, v3.16b, v5.16b
1116
	str	q4, [x1], #16
1117
	str	q2, [x1], #16
1118
	str	q0, [x1], #16
1119
	str	q1, [x1]
1120
	b	.Lcbc_dec_done
1121
.align	4
1122
.Lcbc_dec_six:
1123
	sub	x0, x0, #0x60
1124
	bl	_bsaes_decrypt8
1125
	ldr	q3, [x0], #16               // reload input
1126
	eor	v0.16b, v0.16b, v15.16b     // ^= IV
1127
	ldr	q5, [x0], #16
1128
	ldr	q8, [x0], #16
1129
	ldr	q9, [x0], #16
1130
	str	q0, [x1], #16               // write output
1131
	ldr	q0, [x0], #16
1132
	eor	v1.16b, v1.16b, v3.16b
1133
	ldr	q15, [x0]
1134
	eor	v3.16b, v6.16b, v5.16b
1135
	eor	v4.16b, v4.16b, v8.16b
1136
	eor	v2.16b, v2.16b, v9.16b
1137
	str	q1, [x1], #16
1138
	eor	v0.16b, v7.16b, v0.16b
1139
	str	q3, [x1], #16
1140
	str	q4, [x1], #16
1141
	str	q2, [x1], #16
1142
	str	q0, [x1]
1143
	b	.Lcbc_dec_done
1144
.align	4
1145
.Lcbc_dec_five:
1146
	sub	x0, x0, #0x50
1147
	bl	_bsaes_decrypt8
1148
	ldr	q3, [x0], #16               // reload input
1149
	eor	v0.16b, v0.16b, v15.16b     // ^= IV
1150
	ldr	q5, [x0], #16
1151
	ldr	q7, [x0], #16
1152
	ldr	q8, [x0], #16
1153
	str	q0, [x1], #16               // write output
1154
	ldr	q15, [x0]
1155
	eor	v0.16b, v1.16b, v3.16b
1156
	eor	v1.16b, v6.16b, v5.16b
1157
	eor	v3.16b, v4.16b, v7.16b
1158
	str	q0, [x1], #16
1159
	eor	v0.16b, v2.16b, v8.16b
1160
	str	q1, [x1], #16
1161
	str	q3, [x1], #16
1162
	str	q0, [x1]
1163
	b	.Lcbc_dec_done
1164
.align	4
1165
.Lcbc_dec_four:
1166
	sub	x0, x0, #0x40
1167
	bl	_bsaes_decrypt8
1168
	ldr	q2, [x0], #16               // reload input
1169
	eor	v0.16b, v0.16b, v15.16b     // ^= IV
1170
	ldr	q3, [x0], #16
1171
	ldr	q5, [x0], #16
1172
	str	q0, [x1], #16               // write output
1173
	ldr	q15, [x0]
1174
	eor	v0.16b, v1.16b, v2.16b
1175
	eor	v1.16b, v6.16b, v3.16b
1176
	eor	v2.16b, v4.16b, v5.16b
1177
	str	q0, [x1], #16
1178
	str	q1, [x1], #16
1179
	str	q2, [x1]
1180
	b	.Lcbc_dec_done
1181
.align	4
1182
.Lcbc_dec_three:
1183
	sub	x0, x0, #0x30
1184
	bl	_bsaes_decrypt8
1185
	ldr	q2, [x0], #16               // reload input
1186
	eor	v0.16b, v0.16b, v15.16b     // ^= IV
1187
	ldr	q3, [x0], #16
1188
	ldr	q15, [x0]
1189
	str	q0, [x1], #16               // write output
1190
	eor	v0.16b, v1.16b, v2.16b
1191
	eor	v1.16b, v6.16b, v3.16b
1192
	str	q0, [x1], #16
1193
	str	q1, [x1]
1194
	b	.Lcbc_dec_done
1195
.align	4
1196
.Lcbc_dec_two:
1197
	sub	x0, x0, #0x20
1198
	bl	_bsaes_decrypt8
1199
	ldr	q2, [x0], #16               // reload input
1200
	eor	v0.16b, v0.16b, v15.16b     // ^= IV
1201
	ldr	q15, [x0]
1202
	str	q0, [x1], #16               // write output
1203
	eor	v0.16b, v1.16b, v2.16b
1204
	str	q0, [x1]
1205
	b	.Lcbc_dec_done
1206
.align	4
1207
.Lcbc_dec_one:
1208
	sub	x0, x0, #0x10
1209
	stp	x1, x4, [sp, #-32]!
1210
	str	x14, [sp, #16]
1211
	mov	v8.16b, v15.16b
1212
	mov	v15.16b, v0.16b
1213
	mov	x2, x3
1214
	bl	AES_decrypt
1215
	ldr	x14, [sp, #16]
1216
	ldp	x1, x4, [sp], #32
1217
	ldr	q0, [x1]                    // load result
1218
	eor	v0.16b, v0.16b, v8.16b      // ^= IV
1219
	str	q0, [x1]                    // write output
1220

1221
.align	4
1222
.Lcbc_dec_done:
1223
	movi	v0.16b, #0
1224
	movi	v1.16b, #0
1225
.Lcbc_dec_bzero:	//	wipe key schedule [if any]
1226
	stp	q0, q1, [sp], #32
1227
	cmp	sp, x14
1228
	bne	.Lcbc_dec_bzero
1229
	str	q15, [x4]                   // return IV
1230
	ldp	d8, d9, [sp, #16]
1231
	ldp	d10, d15, [sp, #32]
1232
	ldp	x29, x30, [sp], #48
1233
	ret
1234
.size	ossl_bsaes_cbc_encrypt,.-ossl_bsaes_cbc_encrypt
1235

1236
.globl	ossl_bsaes_ctr32_encrypt_blocks
1237
.type	ossl_bsaes_ctr32_encrypt_blocks,%function
1238
.align	4
1239
// On entry:
1240
//   x0 -> input text (whole 16-byte blocks)
1241
//   x1 -> output text (whole 16-byte blocks)
1242
//   x2 = number of 16-byte blocks to encrypt/decrypt (> 0)
1243
//   x3 -> key
1244
//   x4 -> initial value of 128-bit counter (stored big-endian) which increments, modulo 2^32, for each block
1245
// On exit:
1246
//   Output text filled in
1247
//   No output registers, usual AAPCS64 register preservation
1248
ossl_bsaes_ctr32_encrypt_blocks:
1249
	AARCH64_VALID_CALL_TARGET
1250
	cmp	x2, #8                      // use plain AES for
1251
	blo	.Lctr_enc_short             // small sizes
1252

1253
	stp	x29, x30, [sp, #-80]!
1254
	stp	d8, d9, [sp, #16]
1255
	stp	d10, d11, [sp, #32]
1256
	stp	d12, d13, [sp, #48]
1257
	stp	d14, d15, [sp, #64]
1258

1259
	ldr	w15, [x3, #240]             // get # of rounds
1260
	mov	x14, sp
1261

1262
        // allocate the key schedule on the stack
1263
	add	x17, sp, #96
1264
	sub	x17, x17, x15, lsl #7       // 128 bytes per inner round key, less 96 bytes
1265

1266
        // populate the key schedule
1267
	mov	x9, x3                      // pass key
1268
	mov	x10, x15                    // pass # of rounds
1269
	mov	sp, x17                     // sp is sp
1270
	bl	_bsaes_key_convert
1271
	eor	v7.16b, v7.16b, v15.16b     // fix up last round key
1272
	str	q7, [x17]                   // save last round key
1273

1274
	ldr	q0, [x4]                    // load counter
1275
	add	x13, x11, #.LREVM0SR-.LM0_bigendian
1276
	ldr	q4, [sp]                    // load round0 key
1277

1278
	movi	v8.4s, #1                   // compose 1<<96
1279
	movi	v9.16b, #0
1280
	rev32	v15.16b, v0.16b
1281
	rev32	v0.16b, v0.16b
1282
	ext	v11.16b, v9.16b, v8.16b, #4
1283
	rev32	v4.16b, v4.16b
1284
	add	v12.4s, v11.4s, v11.4s      // compose 2<<96
1285
	str	q4, [sp]                    // save adjusted round0 key
1286
	add	v13.4s, v11.4s, v12.4s      // compose 3<<96
1287
	add	v14.4s, v12.4s, v12.4s      // compose 4<<96
1288
	b	.Lctr_enc_loop
1289

1290
.align	4
1291
.Lctr_enc_loop:
1292
        // Intermix prologue from _bsaes_encrypt8 to use the opportunity
1293
        // to flip byte order in 32-bit counter
1294

1295
	add	v1.4s, v15.4s, v11.4s       // +1
1296
	add	x9, sp, #0x10               // pass next round key
1297
	add	v2.4s, v15.4s, v12.4s       // +2
1298
	ldr	q9, [x13]                   // .LREVM0SR
1299
	ldr	q8, [sp]                    // load round0 key
1300
	add	v3.4s, v15.4s, v13.4s       // +3
1301
	mov	x10, x15                    // pass rounds
1302
	sub	x11, x13, #.LREVM0SR-.LSR   // pass constants
1303
	add	v6.4s, v2.4s, v14.4s
1304
	add	v4.4s, v15.4s, v14.4s       // +4
1305
	add	v7.4s, v3.4s, v14.4s
1306
	add	v15.4s, v4.4s, v14.4s       // next counter
1307
	add	v5.4s, v1.4s, v14.4s
1308

1309
	bl	_bsaes_encrypt8_alt
1310

1311
	subs	x2, x2, #8
1312
	blo	.Lctr_enc_loop_done
1313

1314
	ldr	q16, [x0], #16
1315
	ldr	q17, [x0], #16
1316
	eor	v1.16b, v1.16b, v17.16b
1317
	ldr	q17, [x0], #16
1318
	eor	v0.16b, v0.16b, v16.16b
1319
	eor	v4.16b, v4.16b, v17.16b
1320
	str	q0, [x1], #16
1321
	ldr	q16, [x0], #16
1322
	str	q1, [x1], #16
1323
	mov	v0.16b, v15.16b
1324
	str	q4, [x1], #16
1325
	ldr	q1, [x0], #16
1326
	eor	v4.16b, v6.16b, v16.16b
1327
	eor	v1.16b, v3.16b, v1.16b
1328
	ldr	q3, [x0], #16
1329
	eor	v3.16b, v7.16b, v3.16b
1330
	ldr	q6, [x0], #16
1331
	eor	v2.16b, v2.16b, v6.16b
1332
	ldr	q6, [x0], #16
1333
	eor	v5.16b, v5.16b, v6.16b
1334
	str	q4, [x1], #16
1335
	str	q1, [x1], #16
1336
	str	q3, [x1], #16
1337
	str	q2, [x1], #16
1338
	str	q5, [x1], #16
1339

1340
	bne	.Lctr_enc_loop
1341
	b	.Lctr_enc_done
1342

1343
.align	4
1344
.Lctr_enc_loop_done:
1345
	add	x2, x2, #8
1346
	ldr	q16, [x0], #16              // load input
1347
	eor	v0.16b, v0.16b, v16.16b
1348
	str	q0, [x1], #16               // write output
1349
	cmp	x2, #2
1350
	blo	.Lctr_enc_done
1351
	ldr	q17, [x0], #16
1352
	eor	v1.16b, v1.16b, v17.16b
1353
	str	q1, [x1], #16
1354
	beq	.Lctr_enc_done
1355
	ldr	q18, [x0], #16
1356
	eor	v4.16b, v4.16b, v18.16b
1357
	str	q4, [x1], #16
1358
	cmp	x2, #4
1359
	blo	.Lctr_enc_done
1360
	ldr	q19, [x0], #16
1361
	eor	v6.16b, v6.16b, v19.16b
1362
	str	q6, [x1], #16
1363
	beq	.Lctr_enc_done
1364
	ldr	q20, [x0], #16
1365
	eor	v3.16b, v3.16b, v20.16b
1366
	str	q3, [x1], #16
1367
	cmp	x2, #6
1368
	blo	.Lctr_enc_done
1369
	ldr	q21, [x0], #16
1370
	eor	v7.16b, v7.16b, v21.16b
1371
	str	q7, [x1], #16
1372
	beq	.Lctr_enc_done
1373
	ldr	q22, [x0]
1374
	eor	v2.16b, v2.16b, v22.16b
1375
	str	q2, [x1], #16
1376

1377
.Lctr_enc_done:
1378
	movi	v0.16b, #0
1379
	movi	v1.16b, #0
1380
.Lctr_enc_bzero:	//	wipe key schedule [if any]
1381
	stp	q0, q1, [sp], #32
1382
	cmp	sp, x14
1383
	bne	.Lctr_enc_bzero
1384

1385
	ldp	d8, d9, [sp, #16]
1386
	ldp	d10, d11, [sp, #32]
1387
	ldp	d12, d13, [sp, #48]
1388
	ldp	d14, d15, [sp, #64]
1389
	ldp	x29, x30, [sp], #80
1390
	ret
1391

1392
.Lctr_enc_short:
1393
	stp	x29, x30, [sp, #-96]!
1394
	stp	x19, x20, [sp, #16]
1395
	stp	x21, x22, [sp, #32]
1396
	str	x23, [sp, #48]
1397

1398
	mov	x19, x0                     // copy arguments
1399
	mov	x20, x1
1400
	mov	x21, x2
1401
	mov	x22, x3
1402
	ldr	w23, [x4, #12]              // load counter .LSW
1403
	ldr	q1, [x4]                    // load whole counter value
1404
#ifdef __AARCH64EL__
1405
	rev	w23, w23
1406
#endif
1407
	str	q1, [sp, #80]               // copy counter value
1408

1409
.Lctr_enc_short_loop:
1410
	add	x0, sp, #80                 // input counter value
1411
	add	x1, sp, #64                 // output on the stack
1412
	mov	x2, x22                     // key
1413

1414
	bl	AES_encrypt
1415

1416
	ldr	q0, [x19], #16              // load input
1417
	ldr	q1, [sp, #64]               // load encrypted counter
1418
	add	x23, x23, #1
1419
#ifdef __AARCH64EL__
1420
	rev	w0, w23
1421
	str	w0, [sp, #80+12]            // next counter value
1422
#else
1423
	str	w23, [sp, #80+12]           // next counter value
1424
#endif
1425
	eor	v0.16b, v0.16b, v1.16b
1426
	str	q0, [x20], #16              // store output
1427
	subs	x21, x21, #1
1428
	bne	.Lctr_enc_short_loop
1429

1430
	movi	v0.16b, #0
1431
	movi	v1.16b, #0
1432
	stp	q0, q1, [sp, #64]
1433

1434
	ldr	x23, [sp, #48]
1435
	ldp	x21, x22, [sp, #32]
1436
	ldp	x19, x20, [sp, #16]
1437
	ldp	x29, x30, [sp], #96
1438
	ret
1439
.size	ossl_bsaes_ctr32_encrypt_blocks,.-ossl_bsaes_ctr32_encrypt_blocks
1440

1441
.globl	ossl_bsaes_xts_encrypt
1442
.type	ossl_bsaes_xts_encrypt,%function
1443
.align	4
1444
// On entry:
1445
//   x0 -> input plaintext
1446
//   x1 -> output ciphertext
1447
//   x2 -> length of text in bytes (must be at least 16)
1448
//   x3 -> key1 (used to encrypt the XORed plaintext blocks)
1449
//   x4 -> key2 (used to encrypt the initial vector to yield the initial tweak)
1450
//   x5 -> 16-byte initial vector (typically, sector number)
1451
// On exit:
1452
//   Output ciphertext filled in
1453
//   No output registers, usual AAPCS64 register preservation
1454
ossl_bsaes_xts_encrypt:
1455
	AARCH64_VALID_CALL_TARGET
1456
        // Stack layout:
1457
        // sp ->
1458
        //        nrounds*128-96 bytes: key schedule
1459
        // x19 ->
1460
        //        16 bytes: frame record
1461
        //        4*16 bytes: tweak storage across _bsaes_encrypt8
1462
        //        6*8 bytes: storage for 5 callee-saved general-purpose registers
1463
        //        8*8 bytes: storage for 8 callee-saved SIMD registers
1464
	stp	x29, x30, [sp, #-192]!
1465
	stp	x19, x20, [sp, #80]
1466
	stp	x21, x22, [sp, #96]
1467
	str	x23, [sp, #112]
1468
	stp	d8, d9, [sp, #128]
1469
	stp	d10, d11, [sp, #144]
1470
	stp	d12, d13, [sp, #160]
1471
	stp	d14, d15, [sp, #176]
1472

1473
	mov	x19, sp
1474
	mov	x20, x0
1475
	mov	x21, x1
1476
	mov	x22, x2
1477
	mov	x23, x3
1478

1479
        // generate initial tweak
1480
	sub	sp, sp, #16
1481
	mov	x0, x5                      // iv[]
1482
	mov	x1, sp
1483
	mov	x2, x4                      // key2
1484
	bl	AES_encrypt
1485
	ldr	q11, [sp], #16
1486

1487
	ldr	w1, [x23, #240]             // get # of rounds
1488
        // allocate the key schedule on the stack
1489
	add	x17, sp, #96
1490
	sub	x17, x17, x1, lsl #7        // 128 bytes per inner round key, less 96 bytes
1491

1492
        // populate the key schedule
1493
	mov	x9, x23                     // pass key
1494
	mov	x10, x1                     // pass # of rounds
1495
	mov	sp, x17
1496
	bl	_bsaes_key_convert
1497
	eor	v15.16b, v15.16b, v7.16b    // fix up last round key
1498
	str	q15, [x17]                  // save last round key
1499

1500
	subs	x22, x22, #0x80
1501
	blo	.Lxts_enc_short
1502
	b	.Lxts_enc_loop
1503

1504
.align	4
1505
.Lxts_enc_loop:
1506
	ldr	q8, .Lxts_magic
1507
	mov	x10, x1                     // pass rounds
1508
	add	x2, x19, #16
1509
	ldr	q0, [x20], #16
1510
	sshr	v1.2d, v11.2d, #63
1511
	mov	x9, sp                      // pass key schedule
1512
	ldr	q6, .Lxts_magic+16
1513
	add	v2.2d, v11.2d, v11.2d
1514
	cmtst	v3.2d, v11.2d, v6.2d
1515
	and	v1.16b, v1.16b, v8.16b
1516
	ext	v1.16b, v1.16b, v1.16b, #8
1517
	and	v3.16b, v3.16b, v8.16b
1518
	ldr	q4, [x20], #16
1519
	eor	v12.16b, v2.16b, v1.16b
1520
	eor	v1.16b, v4.16b, v12.16b
1521
	eor	v0.16b, v0.16b, v11.16b
1522
	cmtst	v2.2d, v12.2d, v6.2d
1523
	add	v4.2d, v12.2d, v12.2d
1524
	add	x0, x19, #16
1525
	ext	v3.16b, v3.16b, v3.16b, #8
1526
	and	v2.16b, v2.16b, v8.16b
1527
	eor	v13.16b, v4.16b, v3.16b
1528
	ldr	q3, [x20], #16
1529
	ext	v4.16b, v2.16b, v2.16b, #8
1530
	eor	v2.16b, v3.16b, v13.16b
1531
	ldr	q3, [x20], #16
1532
	add	v5.2d, v13.2d, v13.2d
1533
	cmtst	v7.2d, v13.2d, v6.2d
1534
	and	v7.16b, v7.16b, v8.16b
1535
	ldr	q9, [x20], #16
1536
	ext	v7.16b, v7.16b, v7.16b, #8
1537
	ldr	q10, [x20], #16
1538
	eor	v14.16b, v5.16b, v4.16b
1539
	ldr	q16, [x20], #16
1540
	add	v4.2d, v14.2d, v14.2d
1541
	eor	v3.16b, v3.16b, v14.16b
1542
	eor	v15.16b, v4.16b, v7.16b
1543
	add	v5.2d, v15.2d, v15.2d
1544
	ldr	q7, [x20], #16
1545
	cmtst	v4.2d, v14.2d, v6.2d
1546
	and	v17.16b, v4.16b, v8.16b
1547
	cmtst	v18.2d, v15.2d, v6.2d
1548
	eor	v4.16b, v9.16b, v15.16b
1549
	ext	v9.16b, v17.16b, v17.16b, #8
1550
	eor	v9.16b, v5.16b, v9.16b
1551
	add	v17.2d, v9.2d, v9.2d
1552
	and	v18.16b, v18.16b, v8.16b
1553
	eor	v5.16b, v10.16b, v9.16b
1554
	str	q9, [x2], #16
1555
	ext	v10.16b, v18.16b, v18.16b, #8
1556
	cmtst	v9.2d, v9.2d, v6.2d
1557
	and	v9.16b, v9.16b, v8.16b
1558
	eor	v10.16b, v17.16b, v10.16b
1559
	cmtst	v17.2d, v10.2d, v6.2d
1560
	eor	v6.16b, v16.16b, v10.16b
1561
	str	q10, [x2], #16
1562
	ext	v9.16b, v9.16b, v9.16b, #8
1563
	add	v10.2d, v10.2d, v10.2d
1564
	eor	v9.16b, v10.16b, v9.16b
1565
	str	q9, [x2], #16
1566
	eor	v7.16b, v7.16b, v9.16b
1567
	add	v9.2d, v9.2d, v9.2d
1568
	and	v8.16b, v17.16b, v8.16b
1569
	ext	v8.16b, v8.16b, v8.16b, #8
1570
	eor	v8.16b, v9.16b, v8.16b
1571
	str	q8, [x2]                    // next round tweak
1572

1573
	bl	_bsaes_encrypt8
1574

1575
	ldr	q8, [x0], #16
1576
	eor	v0.16b, v0.16b, v11.16b
1577
	eor	v1.16b, v1.16b, v12.16b
1578
	ldr	q9, [x0], #16
1579
	eor	v4.16b, v4.16b, v13.16b
1580
	eor	v6.16b, v6.16b, v14.16b
1581
	ldr	q10, [x0], #16
1582
	eor	v3.16b, v3.16b, v15.16b
1583
	subs	x22, x22, #0x80
1584
	str	q0, [x21], #16
1585
	ldr	q11, [x0]                   // next round tweak
1586
	str	q1, [x21], #16
1587
	eor	v0.16b, v7.16b, v8.16b
1588
	eor	v1.16b, v2.16b, v9.16b
1589
	str	q4, [x21], #16
1590
	eor	v2.16b, v5.16b, v10.16b
1591
	str	q6, [x21], #16
1592
	str	q3, [x21], #16
1593
	str	q0, [x21], #16
1594
	str	q1, [x21], #16
1595
	str	q2, [x21], #16
1596
	bpl	.Lxts_enc_loop
1597

1598
.Lxts_enc_short:
1599
	adds	x22, x22, #0x70
1600
	bmi	.Lxts_enc_done
1601

1602
	ldr	q8, .Lxts_magic
1603
	sshr	v1.2d, v11.2d, #63
1604
	add	v2.2d, v11.2d, v11.2d
1605
	ldr	q9, .Lxts_magic+16
1606
	subs	x22, x22, #0x10
1607
	ldr	q0, [x20], #16
1608
	and	v1.16b, v1.16b, v8.16b
1609
	cmtst	v3.2d, v11.2d, v9.2d
1610
	ext	v1.16b, v1.16b, v1.16b, #8
1611
	and	v3.16b, v3.16b, v8.16b
1612
	eor	v12.16b, v2.16b, v1.16b
1613
	ext	v1.16b, v3.16b, v3.16b, #8
1614
	add	v2.2d, v12.2d, v12.2d
1615
	cmtst	v3.2d, v12.2d, v9.2d
1616
	eor	v13.16b, v2.16b, v1.16b
1617
	and	v22.16b, v3.16b, v8.16b
1618
	bmi	.Lxts_enc_1
1619

1620
	ext	v2.16b, v22.16b, v22.16b, #8
1621
	add	v3.2d, v13.2d, v13.2d
1622
	ldr	q1, [x20], #16
1623
	cmtst	v4.2d, v13.2d, v9.2d
1624
	subs	x22, x22, #0x10
1625
	eor	v14.16b, v3.16b, v2.16b
1626
	and	v23.16b, v4.16b, v8.16b
1627
	bmi	.Lxts_enc_2
1628

1629
	ext	v3.16b, v23.16b, v23.16b, #8
1630
	add	v4.2d, v14.2d, v14.2d
1631
	ldr	q2, [x20], #16
1632
	cmtst	v5.2d, v14.2d, v9.2d
1633
	eor	v0.16b, v0.16b, v11.16b
1634
	subs	x22, x22, #0x10
1635
	eor	v15.16b, v4.16b, v3.16b
1636
	and	v24.16b, v5.16b, v8.16b
1637
	bmi	.Lxts_enc_3
1638

1639
	ext	v4.16b, v24.16b, v24.16b, #8
1640
	add	v5.2d, v15.2d, v15.2d
1641
	ldr	q3, [x20], #16
1642
	cmtst	v6.2d, v15.2d, v9.2d
1643
	eor	v1.16b, v1.16b, v12.16b
1644
	subs	x22, x22, #0x10
1645
	eor	v16.16b, v5.16b, v4.16b
1646
	and	v25.16b, v6.16b, v8.16b
1647
	bmi	.Lxts_enc_4
1648

1649
	ext	v5.16b, v25.16b, v25.16b, #8
1650
	add	v6.2d, v16.2d, v16.2d
1651
	add	x0, x19, #16
1652
	cmtst	v7.2d, v16.2d, v9.2d
1653
	ldr	q4, [x20], #16
1654
	eor	v2.16b, v2.16b, v13.16b
1655
	str	q16, [x0], #16
1656
	subs	x22, x22, #0x10
1657
	eor	v17.16b, v6.16b, v5.16b
1658
	and	v26.16b, v7.16b, v8.16b
1659
	bmi	.Lxts_enc_5
1660

1661
	ext	v7.16b, v26.16b, v26.16b, #8
1662
	add	v18.2d, v17.2d, v17.2d
1663
	ldr	q5, [x20], #16
1664
	eor	v3.16b, v3.16b, v14.16b
1665
	str	q17, [x0], #16
1666
	subs	x22, x22, #0x10
1667
	eor	v18.16b, v18.16b, v7.16b
1668
	bmi	.Lxts_enc_6
1669

1670
	ldr	q6, [x20], #16
1671
	eor	v4.16b, v4.16b, v15.16b
1672
	eor	v5.16b, v5.16b, v16.16b
1673
	str	q18, [x0]                   // next round tweak
1674
	mov	x9, sp                      // pass key schedule
1675
	mov	x10, x1
1676
	add	x0, x19, #16
1677
	sub	x22, x22, #0x10
1678
	eor	v6.16b, v6.16b, v17.16b
1679

1680
	bl	_bsaes_encrypt8
1681

1682
	ldr	q16, [x0], #16
1683
	eor	v0.16b, v0.16b, v11.16b
1684
	eor	v1.16b, v1.16b, v12.16b
1685
	ldr	q17, [x0], #16
1686
	eor	v4.16b, v4.16b, v13.16b
1687
	eor	v6.16b, v6.16b, v14.16b
1688
	eor	v3.16b, v3.16b, v15.16b
1689
	ldr	q11, [x0]                   // next round tweak
1690
	str	q0, [x21], #16
1691
	str	q1, [x21], #16
1692
	eor	v0.16b, v7.16b, v16.16b
1693
	eor	v1.16b, v2.16b, v17.16b
1694
	str	q4, [x21], #16
1695
	str	q6, [x21], #16
1696
	str	q3, [x21], #16
1697
	str	q0, [x21], #16
1698
	str	q1, [x21], #16
1699
	b	.Lxts_enc_done
1700

1701
.align	4
1702
.Lxts_enc_6:
1703
	eor	v4.16b, v4.16b, v15.16b
1704
	eor	v5.16b, v5.16b, v16.16b
1705
	mov	x9, sp                      // pass key schedule
1706
	mov	x10, x1                     // pass rounds
1707
	add	x0, x19, #16
1708

1709
	bl	_bsaes_encrypt8
1710

1711
	ldr	q16, [x0], #16
1712
	eor	v0.16b, v0.16b, v11.16b
1713
	eor	v1.16b, v1.16b, v12.16b
1714
	eor	v4.16b, v4.16b, v13.16b
1715
	eor	v6.16b, v6.16b, v14.16b
1716
	ldr	q11, [x0]                   // next round tweak
1717
	eor	v3.16b, v3.16b, v15.16b
1718
	str	q0, [x21], #16
1719
	str	q1, [x21], #16
1720
	eor	v0.16b, v7.16b, v16.16b
1721
	str	q4, [x21], #16
1722
	str	q6, [x21], #16
1723
	str	q3, [x21], #16
1724
	str	q0, [x21], #16
1725
	b	.Lxts_enc_done
1726

1727
.align	4
1728
.Lxts_enc_5:
1729
	eor	v3.16b, v3.16b, v14.16b
1730
	eor	v4.16b, v4.16b, v15.16b
1731
	mov	x9, sp                      // pass key schedule
1732
	mov	x10, x1                     // pass rounds
1733
	add	x0, x19, #16
1734

1735
	bl	_bsaes_encrypt8
1736

1737
	eor	v0.16b, v0.16b, v11.16b
1738
	eor	v1.16b, v1.16b, v12.16b
1739
	ldr	q11, [x0]                   // next round tweak
1740
	eor	v4.16b, v4.16b, v13.16b
1741
	eor	v6.16b, v6.16b, v14.16b
1742
	eor	v3.16b, v3.16b, v15.16b
1743
	str	q0, [x21], #16
1744
	str	q1, [x21], #16
1745
	str	q4, [x21], #16
1746
	str	q6, [x21], #16
1747
	str	q3, [x21], #16
1748
	b	.Lxts_enc_done
1749

1750
.align	4
1751
.Lxts_enc_4:
1752
	eor	v2.16b, v2.16b, v13.16b
1753
	eor	v3.16b, v3.16b, v14.16b
1754
	mov	x9, sp                      // pass key schedule
1755
	mov	x10, x1                     // pass rounds
1756
	add	x0, x19, #16
1757

1758
	bl	_bsaes_encrypt8
1759

1760
	eor	v0.16b, v0.16b, v11.16b
1761
	eor	v1.16b, v1.16b, v12.16b
1762
	eor	v4.16b, v4.16b, v13.16b
1763
	eor	v6.16b, v6.16b, v14.16b
1764
	mov	v11.16b, v15.16b            // next round tweak
1765
	str	q0, [x21], #16
1766
	str	q1, [x21], #16
1767
	str	q4, [x21], #16
1768
	str	q6, [x21], #16
1769
	b	.Lxts_enc_done
1770

1771
.align	4
1772
.Lxts_enc_3:
1773
	eor	v1.16b, v1.16b, v12.16b
1774
	eor	v2.16b, v2.16b, v13.16b
1775
	mov	x9, sp                      // pass key schedule
1776
	mov	x10, x1                     // pass rounds
1777
	add	x0, x19, #16
1778

1779
	bl	_bsaes_encrypt8
1780

1781
	eor	v0.16b, v0.16b, v11.16b
1782
	eor	v1.16b, v1.16b, v12.16b
1783
	eor	v4.16b, v4.16b, v13.16b
1784
	mov	v11.16b, v14.16b            // next round tweak
1785
	str	q0, [x21], #16
1786
	str	q1, [x21], #16
1787
	str	q4, [x21], #16
1788
	b	.Lxts_enc_done
1789

1790
.align	4
1791
.Lxts_enc_2:
1792
	eor	v0.16b, v0.16b, v11.16b
1793
	eor	v1.16b, v1.16b, v12.16b
1794
	mov	x9, sp                      // pass key schedule
1795
	mov	x10, x1                     // pass rounds
1796
	add	x0, x19, #16
1797

1798
	bl	_bsaes_encrypt8
1799

1800
	eor	v0.16b, v0.16b, v11.16b
1801
	eor	v1.16b, v1.16b, v12.16b
1802
	mov	v11.16b, v13.16b            // next round tweak
1803
	str	q0, [x21], #16
1804
	str	q1, [x21], #16
1805
	b	.Lxts_enc_done
1806

1807
.align	4
1808
.Lxts_enc_1:
1809
	eor	v0.16b, v0.16b, v11.16b
1810
	sub	x0, sp, #16
1811
	sub	x1, sp, #16
1812
	mov	x2, x23
1813
	mov	v13.d[0], v11.d[1]          // just in case AES_encrypt corrupts top half of callee-saved SIMD registers
1814
	mov	v14.d[0], v12.d[1]
1815
	str	q0, [sp, #-16]!
1816

1817
	bl	AES_encrypt
1818

1819
	ldr	q0, [sp], #16
1820
	trn1	v13.2d, v11.2d, v13.2d
1821
	trn1	v11.2d, v12.2d, v14.2d      // next round tweak
1822
	eor	v0.16b, v0.16b, v13.16b
1823
	str	q0, [x21], #16
1824

1825
.Lxts_enc_done:
1826
	adds	x22, x22, #0x10
1827
	beq	.Lxts_enc_ret
1828

1829
	sub	x6, x21, #0x10
1830
        // Penultimate plaintext block produces final ciphertext part-block
1831
        // plus remaining part of final plaintext block. Move ciphertext part
1832
        // to final position and reuse penultimate ciphertext block buffer to
1833
        // construct final plaintext block
1834
.Lxts_enc_steal:
1835
	ldrb	w0, [x20], #1
1836
	ldrb	w1, [x21, #-0x10]
1837
	strb	w0, [x21, #-0x10]
1838
	strb	w1, [x21], #1
1839

1840
	subs	x22, x22, #1
1841
	bhi	.Lxts_enc_steal
1842

1843
        // Finally encrypt the penultimate ciphertext block using the
1844
        // last tweak
1845
	ldr	q0, [x6]
1846
	eor	v0.16b, v0.16b, v11.16b
1847
	str	q0, [sp, #-16]!
1848
	mov	x0, sp
1849
	mov	x1, sp
1850
	mov	x2, x23
1851
	mov	x21, x6
1852
	mov	v13.d[0], v11.d[1]          // just in case AES_encrypt corrupts top half of callee-saved SIMD registers
1853

1854
	bl	AES_encrypt
1855

1856
	trn1	v11.2d, v11.2d, v13.2d
1857
	ldr	q0, [sp], #16
1858
	eor	v0.16b, v0.16b, v11.16b
1859
	str	q0, [x21]
1860

1861
.Lxts_enc_ret:
1862

1863
	movi	v0.16b, #0
1864
	movi	v1.16b, #0
1865
.Lxts_enc_bzero:	//	wipe key schedule
1866
	stp	q0, q1, [sp], #32
1867
	cmp	sp, x19
1868
	bne	.Lxts_enc_bzero
1869

1870
	ldp	x19, x20, [sp, #80]
1871
	ldp	x21, x22, [sp, #96]
1872
	ldr	x23, [sp, #112]
1873
	ldp	d8, d9, [sp, #128]
1874
	ldp	d10, d11, [sp, #144]
1875
	ldp	d12, d13, [sp, #160]
1876
	ldp	d14, d15, [sp, #176]
1877
	ldp	x29, x30, [sp], #192
1878
	ret
1879
.size	ossl_bsaes_xts_encrypt,.-ossl_bsaes_xts_encrypt
1880

1881
// The assembler doesn't seem capable of de-duplicating these when expressed
1882
// using `ldr qd,=` syntax, so assign a symbolic address
1883
.align	5
1884
.Lxts_magic:
1885
.quad	1, 0x87, 0x4000000000000000, 0x4000000000000000
1886

1887
.globl	ossl_bsaes_xts_decrypt
1888
.type	ossl_bsaes_xts_decrypt,%function
1889
.align	4
1890
// On entry:
1891
//   x0 -> input ciphertext
1892
//   x1 -> output plaintext
1893
//   x2 -> length of text in bytes (must be at least 16)
1894
//   x3 -> key1 (used to decrypt the XORed ciphertext blocks)
1895
//   x4 -> key2 (used to encrypt the initial vector to yield the initial tweak)
1896
//   x5 -> 16-byte initial vector (typically, sector number)
1897
// On exit:
1898
//   Output plaintext filled in
1899
//   No output registers, usual AAPCS64 register preservation
1900
ossl_bsaes_xts_decrypt:
1901
	AARCH64_VALID_CALL_TARGET
1902
        // Stack layout:
1903
        // sp ->
1904
        //        nrounds*128-96 bytes: key schedule
1905
        // x19 ->
1906
        //        16 bytes: frame record
1907
        //        4*16 bytes: tweak storage across _bsaes_decrypt8
1908
        //        6*8 bytes: storage for 5 callee-saved general-purpose registers
1909
        //        8*8 bytes: storage for 8 callee-saved SIMD registers
1910
	stp	x29, x30, [sp, #-192]!
1911
	stp	x19, x20, [sp, #80]
1912
	stp	x21, x22, [sp, #96]
1913
	str	x23, [sp, #112]
1914
	stp	d8, d9, [sp, #128]
1915
	stp	d10, d11, [sp, #144]
1916
	stp	d12, d13, [sp, #160]
1917
	stp	d14, d15, [sp, #176]
1918

1919
	mov	x19, sp
1920
	mov	x20, x0
1921
	mov	x21, x1
1922
	mov	x22, x2
1923
	mov	x23, x3
1924

1925
        // generate initial tweak
1926
	sub	sp, sp, #16
1927
	mov	x0, x5                      // iv[]
1928
	mov	x1, sp
1929
	mov	x2, x4                      // key2
1930
	bl	AES_encrypt
1931
	ldr	q11, [sp], #16
1932

1933
	ldr	w1, [x23, #240]             // get # of rounds
1934
        // allocate the key schedule on the stack
1935
	add	x17, sp, #96
1936
	sub	x17, x17, x1, lsl #7        // 128 bytes per inner round key, less 96 bytes
1937

1938
        // populate the key schedule
1939
	mov	x9, x23                     // pass key
1940
	mov	x10, x1                     // pass # of rounds
1941
	mov	sp, x17
1942
	bl	_bsaes_key_convert
1943
	ldr	q6,  [sp]
1944
	str	q15, [x17]                  // save last round key
1945
	eor	v6.16b, v6.16b, v7.16b      // fix up round 0 key (by XORing with 0x63)
1946
	str	q6, [sp]
1947

1948
	sub	x30, x22, #0x10
1949
	tst	x22, #0xf                   // if not multiple of 16
1950
	csel	x22, x30, x22, ne           // subtract another 16 bytes
1951
	subs	x22, x22, #0x80
1952

1953
	blo	.Lxts_dec_short
1954
	b	.Lxts_dec_loop
1955

1956
.align	4
1957
.Lxts_dec_loop:
1958
	ldr	q8, .Lxts_magic
1959
	mov	x10, x1                     // pass rounds
1960
	add	x2, x19, #16
1961
	ldr	q0, [x20], #16
1962
	sshr	v1.2d, v11.2d, #63
1963
	mov	x9, sp                      // pass key schedule
1964
	ldr	q6, .Lxts_magic+16
1965
	add	v2.2d, v11.2d, v11.2d
1966
	cmtst	v3.2d, v11.2d, v6.2d
1967
	and	v1.16b, v1.16b, v8.16b
1968
	ext	v1.16b, v1.16b, v1.16b, #8
1969
	and	v3.16b, v3.16b, v8.16b
1970
	ldr	q4, [x20], #16
1971
	eor	v12.16b, v2.16b, v1.16b
1972
	eor	v1.16b, v4.16b, v12.16b
1973
	eor	v0.16b, v0.16b, v11.16b
1974
	cmtst	v2.2d, v12.2d, v6.2d
1975
	add	v4.2d, v12.2d, v12.2d
1976
	add	x0, x19, #16
1977
	ext	v3.16b, v3.16b, v3.16b, #8
1978
	and	v2.16b, v2.16b, v8.16b
1979
	eor	v13.16b, v4.16b, v3.16b
1980
	ldr	q3, [x20], #16
1981
	ext	v4.16b, v2.16b, v2.16b, #8
1982
	eor	v2.16b, v3.16b, v13.16b
1983
	ldr	q3, [x20], #16
1984
	add	v5.2d, v13.2d, v13.2d
1985
	cmtst	v7.2d, v13.2d, v6.2d
1986
	and	v7.16b, v7.16b, v8.16b
1987
	ldr	q9, [x20], #16
1988
	ext	v7.16b, v7.16b, v7.16b, #8
1989
	ldr	q10, [x20], #16
1990
	eor	v14.16b, v5.16b, v4.16b
1991
	ldr	q16, [x20], #16
1992
	add	v4.2d, v14.2d, v14.2d
1993
	eor	v3.16b, v3.16b, v14.16b
1994
	eor	v15.16b, v4.16b, v7.16b
1995
	add	v5.2d, v15.2d, v15.2d
1996
	ldr	q7, [x20], #16
1997
	cmtst	v4.2d, v14.2d, v6.2d
1998
	and	v17.16b, v4.16b, v8.16b
1999
	cmtst	v18.2d, v15.2d, v6.2d
2000
	eor	v4.16b, v9.16b, v15.16b
2001
	ext	v9.16b, v17.16b, v17.16b, #8
2002
	eor	v9.16b, v5.16b, v9.16b
2003
	add	v17.2d, v9.2d, v9.2d
2004
	and	v18.16b, v18.16b, v8.16b
2005
	eor	v5.16b, v10.16b, v9.16b
2006
	str	q9, [x2], #16
2007
	ext	v10.16b, v18.16b, v18.16b, #8
2008
	cmtst	v9.2d, v9.2d, v6.2d
2009
	and	v9.16b, v9.16b, v8.16b
2010
	eor	v10.16b, v17.16b, v10.16b
2011
	cmtst	v17.2d, v10.2d, v6.2d
2012
	eor	v6.16b, v16.16b, v10.16b
2013
	str	q10, [x2], #16
2014
	ext	v9.16b, v9.16b, v9.16b, #8
2015
	add	v10.2d, v10.2d, v10.2d
2016
	eor	v9.16b, v10.16b, v9.16b
2017
	str	q9, [x2], #16
2018
	eor	v7.16b, v7.16b, v9.16b
2019
	add	v9.2d, v9.2d, v9.2d
2020
	and	v8.16b, v17.16b, v8.16b
2021
	ext	v8.16b, v8.16b, v8.16b, #8
2022
	eor	v8.16b, v9.16b, v8.16b
2023
	str	q8, [x2]                    // next round tweak
2024

2025
	bl	_bsaes_decrypt8
2026

2027
	eor	v6.16b, v6.16b, v13.16b
2028
	eor	v0.16b, v0.16b, v11.16b
2029
	ldr	q8, [x0], #16
2030
	eor	v7.16b, v7.16b, v8.16b
2031
	str	q0, [x21], #16
2032
	eor	v0.16b, v1.16b, v12.16b
2033
	ldr	q1, [x0], #16
2034
	eor	v1.16b, v3.16b, v1.16b
2035
	subs	x22, x22, #0x80
2036
	eor	v2.16b, v2.16b, v15.16b
2037
	eor	v3.16b, v4.16b, v14.16b
2038
	ldr	q4, [x0], #16
2039
	str	q0, [x21], #16
2040
	ldr	q11, [x0]                   // next round tweak
2041
	eor	v0.16b, v5.16b, v4.16b
2042
	str	q6, [x21], #16
2043
	str	q3, [x21], #16
2044
	str	q2, [x21], #16
2045
	str	q7, [x21], #16
2046
	str	q1, [x21], #16
2047
	str	q0, [x21], #16
2048
	bpl	.Lxts_dec_loop
2049

2050
.Lxts_dec_short:
2051
	adds	x22, x22, #0x70
2052
	bmi	.Lxts_dec_done
2053

2054
	ldr	q8, .Lxts_magic
2055
	sshr	v1.2d, v11.2d, #63
2056
	add	v2.2d, v11.2d, v11.2d
2057
	ldr	q9, .Lxts_magic+16
2058
	subs	x22, x22, #0x10
2059
	ldr	q0, [x20], #16
2060
	and	v1.16b, v1.16b, v8.16b
2061
	cmtst	v3.2d, v11.2d, v9.2d
2062
	ext	v1.16b, v1.16b, v1.16b, #8
2063
	and	v3.16b, v3.16b, v8.16b
2064
	eor	v12.16b, v2.16b, v1.16b
2065
	ext	v1.16b, v3.16b, v3.16b, #8
2066
	add	v2.2d, v12.2d, v12.2d
2067
	cmtst	v3.2d, v12.2d, v9.2d
2068
	eor	v13.16b, v2.16b, v1.16b
2069
	and	v22.16b, v3.16b, v8.16b
2070
	bmi	.Lxts_dec_1
2071

2072
	ext	v2.16b, v22.16b, v22.16b, #8
2073
	add	v3.2d, v13.2d, v13.2d
2074
	ldr	q1, [x20], #16
2075
	cmtst	v4.2d, v13.2d, v9.2d
2076
	subs	x22, x22, #0x10
2077
	eor	v14.16b, v3.16b, v2.16b
2078
	and	v23.16b, v4.16b, v8.16b
2079
	bmi	.Lxts_dec_2
2080

2081
	ext	v3.16b, v23.16b, v23.16b, #8
2082
	add	v4.2d, v14.2d, v14.2d
2083
	ldr	q2, [x20], #16
2084
	cmtst	v5.2d, v14.2d, v9.2d
2085
	eor	v0.16b, v0.16b, v11.16b
2086
	subs	x22, x22, #0x10
2087
	eor	v15.16b, v4.16b, v3.16b
2088
	and	v24.16b, v5.16b, v8.16b
2089
	bmi	.Lxts_dec_3
2090

2091
	ext	v4.16b, v24.16b, v24.16b, #8
2092
	add	v5.2d, v15.2d, v15.2d
2093
	ldr	q3, [x20], #16
2094
	cmtst	v6.2d, v15.2d, v9.2d
2095
	eor	v1.16b, v1.16b, v12.16b
2096
	subs	x22, x22, #0x10
2097
	eor	v16.16b, v5.16b, v4.16b
2098
	and	v25.16b, v6.16b, v8.16b
2099
	bmi	.Lxts_dec_4
2100

2101
	ext	v5.16b, v25.16b, v25.16b, #8
2102
	add	v6.2d, v16.2d, v16.2d
2103
	add	x0, x19, #16
2104
	cmtst	v7.2d, v16.2d, v9.2d
2105
	ldr	q4, [x20], #16
2106
	eor	v2.16b, v2.16b, v13.16b
2107
	str	q16, [x0], #16
2108
	subs	x22, x22, #0x10
2109
	eor	v17.16b, v6.16b, v5.16b
2110
	and	v26.16b, v7.16b, v8.16b
2111
	bmi	.Lxts_dec_5
2112

2113
	ext	v7.16b, v26.16b, v26.16b, #8
2114
	add	v18.2d, v17.2d, v17.2d
2115
	ldr	q5, [x20], #16
2116
	eor	v3.16b, v3.16b, v14.16b
2117
	str	q17, [x0], #16
2118
	subs	x22, x22, #0x10
2119
	eor	v18.16b, v18.16b, v7.16b
2120
	bmi	.Lxts_dec_6
2121

2122
	ldr	q6, [x20], #16
2123
	eor	v4.16b, v4.16b, v15.16b
2124
	eor	v5.16b, v5.16b, v16.16b
2125
	str	q18, [x0]                   // next round tweak
2126
	mov	x9, sp                      // pass key schedule
2127
	mov	x10, x1
2128
	add	x0, x19, #16
2129
	sub	x22, x22, #0x10
2130
	eor	v6.16b, v6.16b, v17.16b
2131

2132
	bl	_bsaes_decrypt8
2133

2134
	ldr	q16, [x0], #16
2135
	eor	v0.16b, v0.16b, v11.16b
2136
	eor	v1.16b, v1.16b, v12.16b
2137
	ldr	q17, [x0], #16
2138
	eor	v6.16b, v6.16b, v13.16b
2139
	eor	v4.16b, v4.16b, v14.16b
2140
	eor	v2.16b, v2.16b, v15.16b
2141
	ldr	q11, [x0]                   // next round tweak
2142
	str	q0, [x21], #16
2143
	str	q1, [x21], #16
2144
	eor	v0.16b, v7.16b, v16.16b
2145
	eor	v1.16b, v3.16b, v17.16b
2146
	str	q6, [x21], #16
2147
	str	q4, [x21], #16
2148
	str	q2, [x21], #16
2149
	str	q0, [x21], #16
2150
	str	q1, [x21], #16
2151
	b	.Lxts_dec_done
2152

2153
.align	4
2154
.Lxts_dec_6:
2155
	eor	v4.16b, v4.16b, v15.16b
2156
	eor	v5.16b, v5.16b, v16.16b
2157
	mov	x9, sp                      // pass key schedule
2158
	mov	x10, x1                     // pass rounds
2159
	add	x0, x19, #16
2160

2161
	bl	_bsaes_decrypt8
2162

2163
	ldr	q16, [x0], #16
2164
	eor	v0.16b, v0.16b, v11.16b
2165
	eor	v1.16b, v1.16b, v12.16b
2166
	eor	v6.16b, v6.16b, v13.16b
2167
	eor	v4.16b, v4.16b, v14.16b
2168
	ldr	q11, [x0]                   // next round tweak
2169
	eor	v2.16b, v2.16b, v15.16b
2170
	str	q0, [x21], #16
2171
	str	q1, [x21], #16
2172
	eor	v0.16b, v7.16b, v16.16b
2173
	str	q6, [x21], #16
2174
	str	q4, [x21], #16
2175
	str	q2, [x21], #16
2176
	str	q0, [x21], #16
2177
	b	.Lxts_dec_done
2178

2179
.align	4
2180
.Lxts_dec_5:
2181
	eor	v3.16b, v3.16b, v14.16b
2182
	eor	v4.16b, v4.16b, v15.16b
2183
	mov	x9, sp                      // pass key schedule
2184
	mov	x10, x1                     // pass rounds
2185
	add	x0, x19, #16
2186

2187
	bl	_bsaes_decrypt8
2188

2189
	eor	v0.16b, v0.16b, v11.16b
2190
	eor	v1.16b, v1.16b, v12.16b
2191
	ldr	q11, [x0]                   // next round tweak
2192
	eor	v6.16b, v6.16b, v13.16b
2193
	eor	v4.16b, v4.16b, v14.16b
2194
	eor	v2.16b, v2.16b, v15.16b
2195
	str	q0, [x21], #16
2196
	str	q1, [x21], #16
2197
	str	q6, [x21], #16
2198
	str	q4, [x21], #16
2199
	str	q2, [x21], #16
2200
	b	.Lxts_dec_done
2201

2202
.align	4
2203
.Lxts_dec_4:
2204
	eor	v2.16b, v2.16b, v13.16b
2205
	eor	v3.16b, v3.16b, v14.16b
2206
	mov	x9, sp                      // pass key schedule
2207
	mov	x10, x1                     // pass rounds
2208
	add	x0, x19, #16
2209

2210
	bl	_bsaes_decrypt8
2211

2212
	eor	v0.16b, v0.16b, v11.16b
2213
	eor	v1.16b, v1.16b, v12.16b
2214
	eor	v6.16b, v6.16b, v13.16b
2215
	eor	v4.16b, v4.16b, v14.16b
2216
	mov	v11.16b, v15.16b            // next round tweak
2217
	str	q0, [x21], #16
2218
	str	q1, [x21], #16
2219
	str	q6, [x21], #16
2220
	str	q4, [x21], #16
2221
	b	.Lxts_dec_done
2222

2223
.align	4
2224
.Lxts_dec_3:
2225
	eor	v1.16b, v1.16b, v12.16b
2226
	eor	v2.16b, v2.16b, v13.16b
2227
	mov	x9, sp                      // pass key schedule
2228
	mov	x10, x1                     // pass rounds
2229
	add	x0, x19, #16
2230

2231
	bl	_bsaes_decrypt8
2232

2233
	eor	v0.16b, v0.16b, v11.16b
2234
	eor	v1.16b, v1.16b, v12.16b
2235
	eor	v6.16b, v6.16b, v13.16b
2236
	mov	v11.16b, v14.16b            // next round tweak
2237
	str	q0, [x21], #16
2238
	str	q1, [x21], #16
2239
	str	q6, [x21], #16
2240
	b	.Lxts_dec_done
2241

2242
.align	4
2243
.Lxts_dec_2:
2244
	eor	v0.16b, v0.16b, v11.16b
2245
	eor	v1.16b, v1.16b, v12.16b
2246
	mov	x9, sp                      // pass key schedule
2247
	mov	x10, x1                     // pass rounds
2248
	add	x0, x19, #16
2249

2250
	bl	_bsaes_decrypt8
2251

2252
	eor	v0.16b, v0.16b, v11.16b
2253
	eor	v1.16b, v1.16b, v12.16b
2254
	mov	v11.16b, v13.16b            // next round tweak
2255
	str	q0, [x21], #16
2256
	str	q1, [x21], #16
2257
	b	.Lxts_dec_done
2258

2259
.align	4
2260
.Lxts_dec_1:
2261
	eor	v0.16b, v0.16b, v11.16b
2262
	sub	x0, sp, #16
2263
	sub	x1, sp, #16
2264
	mov	x2, x23
2265
	mov	v13.d[0], v11.d[1]          // just in case AES_decrypt corrupts top half of callee-saved SIMD registers
2266
	mov	v14.d[0], v12.d[1]
2267
	str	q0, [sp, #-16]!
2268

2269
	bl	AES_decrypt
2270

2271
	ldr	q0, [sp], #16
2272
	trn1	v13.2d, v11.2d, v13.2d
2273
	trn1	v11.2d, v12.2d, v14.2d      // next round tweak
2274
	eor	v0.16b, v0.16b, v13.16b
2275
	str	q0, [x21], #16
2276

2277
.Lxts_dec_done:
2278
	adds	x22, x22, #0x10
2279
	beq	.Lxts_dec_ret
2280

2281
        // calculate one round of extra tweak for the stolen ciphertext
2282
	ldr	q8, .Lxts_magic
2283
	sshr	v6.2d, v11.2d, #63
2284
	and	v6.16b, v6.16b, v8.16b
2285
	add	v12.2d, v11.2d, v11.2d
2286
	ext	v6.16b, v6.16b, v6.16b, #8
2287
	eor	v12.16b, v12.16b, v6.16b
2288

2289
        // perform the final decryption with the last tweak value
2290
	ldr	q0, [x20], #16
2291
	eor	v0.16b, v0.16b, v12.16b
2292
	str	q0, [sp, #-16]!
2293
	mov	x0, sp
2294
	mov	x1, sp
2295
	mov	x2, x23
2296
	mov	v13.d[0], v11.d[1]          // just in case AES_decrypt corrupts top half of callee-saved SIMD registers
2297
	mov	v14.d[0], v12.d[1]
2298

2299
	bl	AES_decrypt
2300

2301
	trn1	v12.2d, v12.2d, v14.2d
2302
	trn1	v11.2d, v11.2d, v13.2d
2303
	ldr	q0, [sp], #16
2304
	eor	v0.16b, v0.16b, v12.16b
2305
	str	q0, [x21]
2306

2307
	mov	x6, x21
2308
        // Penultimate ciphertext block produces final plaintext part-block
2309
        // plus remaining part of final ciphertext block. Move plaintext part
2310
        // to final position and reuse penultimate plaintext block buffer to
2311
        // construct final ciphertext block
2312
.Lxts_dec_steal:
2313
	ldrb	w1, [x21]
2314
	ldrb	w0, [x20], #1
2315
	strb	w1, [x21, #0x10]
2316
	strb	w0, [x21], #1
2317

2318
	subs	x22, x22, #1
2319
	bhi	.Lxts_dec_steal
2320

2321
        // Finally decrypt the penultimate plaintext block using the
2322
        // penultimate tweak
2323
	ldr	q0, [x6]
2324
	eor	v0.16b, v0.16b, v11.16b
2325
	str	q0, [sp, #-16]!
2326
	mov	x0, sp
2327
	mov	x1, sp
2328
	mov	x2, x23
2329
	mov	x21, x6
2330

2331
	bl	AES_decrypt
2332

2333
	trn1	v11.2d, v11.2d, v13.2d
2334
	ldr	q0, [sp], #16
2335
	eor	v0.16b, v0.16b, v11.16b
2336
	str	q0, [x21]
2337

2338
.Lxts_dec_ret:
2339

2340
	movi	v0.16b, #0
2341
	movi	v1.16b, #0
2342
.Lxts_dec_bzero:	//	wipe key schedule
2343
	stp	q0, q1, [sp], #32
2344
	cmp	sp, x19
2345
	bne	.Lxts_dec_bzero
2346

2347
	ldp	x19, x20, [sp, #80]
2348
	ldp	x21, x22, [sp, #96]
2349
	ldr	x23, [sp, #112]
2350
	ldp	d8, d9, [sp, #128]
2351
	ldp	d10, d11, [sp, #144]
2352
	ldp	d12, d13, [sp, #160]
2353
	ldp	d14, d15, [sp, #176]
2354
	ldp	x29, x30, [sp], #192
2355
	ret
2356
.size	ossl_bsaes_xts_decrypt,.-ossl_bsaes_xts_decrypt
2357

2358
Product

Resources

Company
