CoCalc -- sha256-armv7.S

GitHub Repository: freebsd/freebsd-src
Path: blob/main/sys/contrib/openzfs/module/icp/asm-arm/sha2/sha256-armv7.S
⁴⁸⁷⁷⁵ views
1
// SPDX-License-Identifier: Apache-2.0
2
/*
3
 * Copyright 2004-2022 The OpenSSL Project Authors. All Rights Reserved.
4
 *
5
 * Licensed under the Apache License, Version 2.0 (the "License");
6
 * you may not use this file except in compliance with the License.
7
 * You may obtain a copy of the License at
8
 *
9
 *     https://www.apache.org/licenses/LICENSE-2.0
10
 *
11
 * Unless required by applicable law or agreed to in writing, software
12
 * distributed under the License is distributed on an "AS IS" BASIS,
13
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
 * See the License for the specific language governing permissions and
15
 * limitations under the License.
16
 */
17

18
/*
19
 * Portions Copyright (c) 2022 Tino Reichardt <[email protected]>
20
 * - modified assembly to fit into OpenZFS
21
 */
22

23
#if defined(__arm__)
24

25
#ifndef __ARM_ARCH
26
# define __ARM_ARCH__	7
27
#else
28
# define __ARM_ARCH__	__ARM_ARCH
29
#endif
30

31
#if defined(__thumb2__)
32
.syntax unified
33
.thumb
34
#else
35
.code   32
36
#endif
37

38
.text
39

40
.type	K256,%object
41
.align	5
42
K256:
43
.word	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
44
.word	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
45
.word	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
46
.word	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
47
.word	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
48
.word	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
49
.word	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
50
.word	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
51
.word	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
52
.word	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
53
.word	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
54
.word	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
55
.word	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
56
.word	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
57
.word	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
58
.word	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
59
.size	K256,.-K256
60
.word	0				@ terminator
61

62
.align	5
63
.globl	zfs_sha256_block_armv7
64
.type	zfs_sha256_block_armv7,%function
65
zfs_sha256_block_armv7:
66
.Lzfs_sha256_block_armv7:
67

68
#if __ARM_ARCH__<7 && !defined(__thumb2__)
69
	sub	r3,pc,#8		@ zfs_sha256_block_armv7
70
#else
71
	adr	r3,.Lzfs_sha256_block_armv7
72
#endif
73

74
	add	r2,r1,r2,lsl#6	@ len to point at the end of inp
75
	stmdb	sp!,{r0,r1,r2,r4-r11,lr}
76
	ldmia	r0,{r4,r5,r6,r7,r8,r9,r10,r11}
77
	sub	r14,r3,#256+32	@ K256
78
	sub	sp,sp,#16*4		@ alloca(X[16])
79
.Loop:
80
# if __ARM_ARCH__>=7
81
	ldr	r2,[r1],#4
82
# else
83
	ldrb	r2,[r1,#3]
84
# endif
85
	eor	r3,r5,r6		@ magic
86
	eor	r12,r12,r12
87
#if __ARM_ARCH__>=7
88
	@ ldr	r2,[r1],#4			@ 0
89
# if 0==15
90
	str	r1,[sp,#17*4]			@ make room for r1
91
# endif
92
	eor	r0,r8,r8,ror#5
93
	add	r4,r4,r12			@ h+=Maj(a,b,c) from the past
94
	eor	r0,r0,r8,ror#19	@ Sigma1(e)
95
# ifndef __ARMEB__
96
	rev	r2,r2
97
# endif
98
#else
99
	@ ldrb	r2,[r1,#3]			@ 0
100
	add	r4,r4,r12			@ h+=Maj(a,b,c) from the past
101
	ldrb	r12,[r1,#2]
102
	ldrb	r0,[r1,#1]
103
	orr	r2,r2,r12,lsl#8
104
	ldrb	r12,[r1],#4
105
	orr	r2,r2,r0,lsl#16
106
# if 0==15
107
	str	r1,[sp,#17*4]			@ make room for r1
108
# endif
109
	eor	r0,r8,r8,ror#5
110
	orr	r2,r2,r12,lsl#24
111
	eor	r0,r0,r8,ror#19	@ Sigma1(e)
112
#endif
113
	ldr	r12,[r14],#4			@ *K256++
114
	add	r11,r11,r2			@ h+=X[i]
115
	str	r2,[sp,#0*4]
116
	eor	r2,r9,r10
117
	add	r11,r11,r0,ror#6	@ h+=Sigma1(e)
118
	and	r2,r2,r8
119
	add	r11,r11,r12			@ h+=K256[i]
120
	eor	r2,r2,r10			@ Ch(e,f,g)
121
	eor	r0,r4,r4,ror#11
122
	add	r11,r11,r2			@ h+=Ch(e,f,g)
123
#if 0==31
124
	and	r12,r12,#0xff
125
	cmp	r12,#0xf2			@ done?
126
#endif
127
#if 0<15
128
# if __ARM_ARCH__>=7
129
	ldr	r2,[r1],#4			@ prefetch
130
# else
131
	ldrb	r2,[r1,#3]
132
# endif
133
	eor	r12,r4,r5			@ a^b, b^c in next round
134
#else
135
	ldr	r2,[sp,#2*4]		@ from future BODY_16_xx
136
	eor	r12,r4,r5			@ a^b, b^c in next round
137
	ldr	r1,[sp,#15*4]	@ from future BODY_16_xx
138
#endif
139
	eor	r0,r0,r4,ror#20	@ Sigma0(a)
140
	and	r3,r3,r12			@ (b^c)&=(a^b)
141
	add	r7,r7,r11			@ d+=h
142
	eor	r3,r3,r5			@ Maj(a,b,c)
143
	add	r11,r11,r0,ror#2	@ h+=Sigma0(a)
144
	@ add	r11,r11,r3			@ h+=Maj(a,b,c)
145
#if __ARM_ARCH__>=7
146
	@ ldr	r2,[r1],#4			@ 1
147
# if 1==15
148
	str	r1,[sp,#17*4]			@ make room for r1
149
# endif
150
	eor	r0,r7,r7,ror#5
151
	add	r11,r11,r3			@ h+=Maj(a,b,c) from the past
152
	eor	r0,r0,r7,ror#19	@ Sigma1(e)
153
# ifndef __ARMEB__
154
	rev	r2,r2
155
# endif
156
#else
157
	@ ldrb	r2,[r1,#3]			@ 1
158
	add	r11,r11,r3			@ h+=Maj(a,b,c) from the past
159
	ldrb	r3,[r1,#2]
160
	ldrb	r0,[r1,#1]
161
	orr	r2,r2,r3,lsl#8
162
	ldrb	r3,[r1],#4
163
	orr	r2,r2,r0,lsl#16
164
# if 1==15
165
	str	r1,[sp,#17*4]			@ make room for r1
166
# endif
167
	eor	r0,r7,r7,ror#5
168
	orr	r2,r2,r3,lsl#24
169
	eor	r0,r0,r7,ror#19	@ Sigma1(e)
170
#endif
171
	ldr	r3,[r14],#4			@ *K256++
172
	add	r10,r10,r2			@ h+=X[i]
173
	str	r2,[sp,#1*4]
174
	eor	r2,r8,r9
175
	add	r10,r10,r0,ror#6	@ h+=Sigma1(e)
176
	and	r2,r2,r7
177
	add	r10,r10,r3			@ h+=K256[i]
178
	eor	r2,r2,r9			@ Ch(e,f,g)
179
	eor	r0,r11,r11,ror#11
180
	add	r10,r10,r2			@ h+=Ch(e,f,g)
181
#if 1==31
182
	and	r3,r3,#0xff
183
	cmp	r3,#0xf2			@ done?
184
#endif
185
#if 1<15
186
# if __ARM_ARCH__>=7
187
	ldr	r2,[r1],#4			@ prefetch
188
# else
189
	ldrb	r2,[r1,#3]
190
# endif
191
	eor	r3,r11,r4			@ a^b, b^c in next round
192
#else
193
	ldr	r2,[sp,#3*4]		@ from future BODY_16_xx
194
	eor	r3,r11,r4			@ a^b, b^c in next round
195
	ldr	r1,[sp,#0*4]	@ from future BODY_16_xx
196
#endif
197
	eor	r0,r0,r11,ror#20	@ Sigma0(a)
198
	and	r12,r12,r3			@ (b^c)&=(a^b)
199
	add	r6,r6,r10			@ d+=h
200
	eor	r12,r12,r4			@ Maj(a,b,c)
201
	add	r10,r10,r0,ror#2	@ h+=Sigma0(a)
202
	@ add	r10,r10,r12			@ h+=Maj(a,b,c)
203
#if __ARM_ARCH__>=7
204
	@ ldr	r2,[r1],#4			@ 2
205
# if 2==15
206
	str	r1,[sp,#17*4]			@ make room for r1
207
# endif
208
	eor	r0,r6,r6,ror#5
209
	add	r10,r10,r12			@ h+=Maj(a,b,c) from the past
210
	eor	r0,r0,r6,ror#19	@ Sigma1(e)
211
# ifndef __ARMEB__
212
	rev	r2,r2
213
# endif
214
#else
215
	@ ldrb	r2,[r1,#3]			@ 2
216
	add	r10,r10,r12			@ h+=Maj(a,b,c) from the past
217
	ldrb	r12,[r1,#2]
218
	ldrb	r0,[r1,#1]
219
	orr	r2,r2,r12,lsl#8
220
	ldrb	r12,[r1],#4
221
	orr	r2,r2,r0,lsl#16
222
# if 2==15
223
	str	r1,[sp,#17*4]			@ make room for r1
224
# endif
225
	eor	r0,r6,r6,ror#5
226
	orr	r2,r2,r12,lsl#24
227
	eor	r0,r0,r6,ror#19	@ Sigma1(e)
228
#endif
229
	ldr	r12,[r14],#4			@ *K256++
230
	add	r9,r9,r2			@ h+=X[i]
231
	str	r2,[sp,#2*4]
232
	eor	r2,r7,r8
233
	add	r9,r9,r0,ror#6	@ h+=Sigma1(e)
234
	and	r2,r2,r6
235
	add	r9,r9,r12			@ h+=K256[i]
236
	eor	r2,r2,r8			@ Ch(e,f,g)
237
	eor	r0,r10,r10,ror#11
238
	add	r9,r9,r2			@ h+=Ch(e,f,g)
239
#if 2==31
240
	and	r12,r12,#0xff
241
	cmp	r12,#0xf2			@ done?
242
#endif
243
#if 2<15
244
# if __ARM_ARCH__>=7
245
	ldr	r2,[r1],#4			@ prefetch
246
# else
247
	ldrb	r2,[r1,#3]
248
# endif
249
	eor	r12,r10,r11			@ a^b, b^c in next round
250
#else
251
	ldr	r2,[sp,#4*4]		@ from future BODY_16_xx
252
	eor	r12,r10,r11			@ a^b, b^c in next round
253
	ldr	r1,[sp,#1*4]	@ from future BODY_16_xx
254
#endif
255
	eor	r0,r0,r10,ror#20	@ Sigma0(a)
256
	and	r3,r3,r12			@ (b^c)&=(a^b)
257
	add	r5,r5,r9			@ d+=h
258
	eor	r3,r3,r11			@ Maj(a,b,c)
259
	add	r9,r9,r0,ror#2	@ h+=Sigma0(a)
260
	@ add	r9,r9,r3			@ h+=Maj(a,b,c)
261
#if __ARM_ARCH__>=7
262
	@ ldr	r2,[r1],#4			@ 3
263
# if 3==15
264
	str	r1,[sp,#17*4]			@ make room for r1
265
# endif
266
	eor	r0,r5,r5,ror#5
267
	add	r9,r9,r3			@ h+=Maj(a,b,c) from the past
268
	eor	r0,r0,r5,ror#19	@ Sigma1(e)
269
# ifndef __ARMEB__
270
	rev	r2,r2
271
# endif
272
#else
273
	@ ldrb	r2,[r1,#3]			@ 3
274
	add	r9,r9,r3			@ h+=Maj(a,b,c) from the past
275
	ldrb	r3,[r1,#2]
276
	ldrb	r0,[r1,#1]
277
	orr	r2,r2,r3,lsl#8
278
	ldrb	r3,[r1],#4
279
	orr	r2,r2,r0,lsl#16
280
# if 3==15
281
	str	r1,[sp,#17*4]			@ make room for r1
282
# endif
283
	eor	r0,r5,r5,ror#5
284
	orr	r2,r2,r3,lsl#24
285
	eor	r0,r0,r5,ror#19	@ Sigma1(e)
286
#endif
287
	ldr	r3,[r14],#4			@ *K256++
288
	add	r8,r8,r2			@ h+=X[i]
289
	str	r2,[sp,#3*4]
290
	eor	r2,r6,r7
291
	add	r8,r8,r0,ror#6	@ h+=Sigma1(e)
292
	and	r2,r2,r5
293
	add	r8,r8,r3			@ h+=K256[i]
294
	eor	r2,r2,r7			@ Ch(e,f,g)
295
	eor	r0,r9,r9,ror#11
296
	add	r8,r8,r2			@ h+=Ch(e,f,g)
297
#if 3==31
298
	and	r3,r3,#0xff
299
	cmp	r3,#0xf2			@ done?
300
#endif
301
#if 3<15
302
# if __ARM_ARCH__>=7
303
	ldr	r2,[r1],#4			@ prefetch
304
# else
305
	ldrb	r2,[r1,#3]
306
# endif
307
	eor	r3,r9,r10			@ a^b, b^c in next round
308
#else
309
	ldr	r2,[sp,#5*4]		@ from future BODY_16_xx
310
	eor	r3,r9,r10			@ a^b, b^c in next round
311
	ldr	r1,[sp,#2*4]	@ from future BODY_16_xx
312
#endif
313
	eor	r0,r0,r9,ror#20	@ Sigma0(a)
314
	and	r12,r12,r3			@ (b^c)&=(a^b)
315
	add	r4,r4,r8			@ d+=h
316
	eor	r12,r12,r10			@ Maj(a,b,c)
317
	add	r8,r8,r0,ror#2	@ h+=Sigma0(a)
318
	@ add	r8,r8,r12			@ h+=Maj(a,b,c)
319
#if __ARM_ARCH__>=7
320
	@ ldr	r2,[r1],#4			@ 4
321
# if 4==15
322
	str	r1,[sp,#17*4]			@ make room for r1
323
# endif
324
	eor	r0,r4,r4,ror#5
325
	add	r8,r8,r12			@ h+=Maj(a,b,c) from the past
326
	eor	r0,r0,r4,ror#19	@ Sigma1(e)
327
# ifndef __ARMEB__
328
	rev	r2,r2
329
# endif
330
#else
331
	@ ldrb	r2,[r1,#3]			@ 4
332
	add	r8,r8,r12			@ h+=Maj(a,b,c) from the past
333
	ldrb	r12,[r1,#2]
334
	ldrb	r0,[r1,#1]
335
	orr	r2,r2,r12,lsl#8
336
	ldrb	r12,[r1],#4
337
	orr	r2,r2,r0,lsl#16
338
# if 4==15
339
	str	r1,[sp,#17*4]			@ make room for r1
340
# endif
341
	eor	r0,r4,r4,ror#5
342
	orr	r2,r2,r12,lsl#24
343
	eor	r0,r0,r4,ror#19	@ Sigma1(e)
344
#endif
345
	ldr	r12,[r14],#4			@ *K256++
346
	add	r7,r7,r2			@ h+=X[i]
347
	str	r2,[sp,#4*4]
348
	eor	r2,r5,r6
349
	add	r7,r7,r0,ror#6	@ h+=Sigma1(e)
350
	and	r2,r2,r4
351
	add	r7,r7,r12			@ h+=K256[i]
352
	eor	r2,r2,r6			@ Ch(e,f,g)
353
	eor	r0,r8,r8,ror#11
354
	add	r7,r7,r2			@ h+=Ch(e,f,g)
355
#if 4==31
356
	and	r12,r12,#0xff
357
	cmp	r12,#0xf2			@ done?
358
#endif
359
#if 4<15
360
# if __ARM_ARCH__>=7
361
	ldr	r2,[r1],#4			@ prefetch
362
# else
363
	ldrb	r2,[r1,#3]
364
# endif
365
	eor	r12,r8,r9			@ a^b, b^c in next round
366
#else
367
	ldr	r2,[sp,#6*4]		@ from future BODY_16_xx
368
	eor	r12,r8,r9			@ a^b, b^c in next round
369
	ldr	r1,[sp,#3*4]	@ from future BODY_16_xx
370
#endif
371
	eor	r0,r0,r8,ror#20	@ Sigma0(a)
372
	and	r3,r3,r12			@ (b^c)&=(a^b)
373
	add	r11,r11,r7			@ d+=h
374
	eor	r3,r3,r9			@ Maj(a,b,c)
375
	add	r7,r7,r0,ror#2	@ h+=Sigma0(a)
376
	@ add	r7,r7,r3			@ h+=Maj(a,b,c)
377
#if __ARM_ARCH__>=7
378
	@ ldr	r2,[r1],#4			@ 5
379
# if 5==15
380
	str	r1,[sp,#17*4]			@ make room for r1
381
# endif
382
	eor	r0,r11,r11,ror#5
383
	add	r7,r7,r3			@ h+=Maj(a,b,c) from the past
384
	eor	r0,r0,r11,ror#19	@ Sigma1(e)
385
# ifndef __ARMEB__
386
	rev	r2,r2
387
# endif
388
#else
389
	@ ldrb	r2,[r1,#3]			@ 5
390
	add	r7,r7,r3			@ h+=Maj(a,b,c) from the past
391
	ldrb	r3,[r1,#2]
392
	ldrb	r0,[r1,#1]
393
	orr	r2,r2,r3,lsl#8
394
	ldrb	r3,[r1],#4
395
	orr	r2,r2,r0,lsl#16
396
# if 5==15
397
	str	r1,[sp,#17*4]			@ make room for r1
398
# endif
399
	eor	r0,r11,r11,ror#5
400
	orr	r2,r2,r3,lsl#24
401
	eor	r0,r0,r11,ror#19	@ Sigma1(e)
402
#endif
403
	ldr	r3,[r14],#4			@ *K256++
404
	add	r6,r6,r2			@ h+=X[i]
405
	str	r2,[sp,#5*4]
406
	eor	r2,r4,r5
407
	add	r6,r6,r0,ror#6	@ h+=Sigma1(e)
408
	and	r2,r2,r11
409
	add	r6,r6,r3			@ h+=K256[i]
410
	eor	r2,r2,r5			@ Ch(e,f,g)
411
	eor	r0,r7,r7,ror#11
412
	add	r6,r6,r2			@ h+=Ch(e,f,g)
413
#if 5==31
414
	and	r3,r3,#0xff
415
	cmp	r3,#0xf2			@ done?
416
#endif
417
#if 5<15
418
# if __ARM_ARCH__>=7
419
	ldr	r2,[r1],#4			@ prefetch
420
# else
421
	ldrb	r2,[r1,#3]
422
# endif
423
	eor	r3,r7,r8			@ a^b, b^c in next round
424
#else
425
	ldr	r2,[sp,#7*4]		@ from future BODY_16_xx
426
	eor	r3,r7,r8			@ a^b, b^c in next round
427
	ldr	r1,[sp,#4*4]	@ from future BODY_16_xx
428
#endif
429
	eor	r0,r0,r7,ror#20	@ Sigma0(a)
430
	and	r12,r12,r3			@ (b^c)&=(a^b)
431
	add	r10,r10,r6			@ d+=h
432
	eor	r12,r12,r8			@ Maj(a,b,c)
433
	add	r6,r6,r0,ror#2	@ h+=Sigma0(a)
434
	@ add	r6,r6,r12			@ h+=Maj(a,b,c)
435
#if __ARM_ARCH__>=7
436
	@ ldr	r2,[r1],#4			@ 6
437
# if 6==15
438
	str	r1,[sp,#17*4]			@ make room for r1
439
# endif
440
	eor	r0,r10,r10,ror#5
441
	add	r6,r6,r12			@ h+=Maj(a,b,c) from the past
442
	eor	r0,r0,r10,ror#19	@ Sigma1(e)
443
# ifndef __ARMEB__
444
	rev	r2,r2
445
# endif
446
#else
447
	@ ldrb	r2,[r1,#3]			@ 6
448
	add	r6,r6,r12			@ h+=Maj(a,b,c) from the past
449
	ldrb	r12,[r1,#2]
450
	ldrb	r0,[r1,#1]
451
	orr	r2,r2,r12,lsl#8
452
	ldrb	r12,[r1],#4
453
	orr	r2,r2,r0,lsl#16
454
# if 6==15
455
	str	r1,[sp,#17*4]			@ make room for r1
456
# endif
457
	eor	r0,r10,r10,ror#5
458
	orr	r2,r2,r12,lsl#24
459
	eor	r0,r0,r10,ror#19	@ Sigma1(e)
460
#endif
461
	ldr	r12,[r14],#4			@ *K256++
462
	add	r5,r5,r2			@ h+=X[i]
463
	str	r2,[sp,#6*4]
464
	eor	r2,r11,r4
465
	add	r5,r5,r0,ror#6	@ h+=Sigma1(e)
466
	and	r2,r2,r10
467
	add	r5,r5,r12			@ h+=K256[i]
468
	eor	r2,r2,r4			@ Ch(e,f,g)
469
	eor	r0,r6,r6,ror#11
470
	add	r5,r5,r2			@ h+=Ch(e,f,g)
471
#if 6==31
472
	and	r12,r12,#0xff
473
	cmp	r12,#0xf2			@ done?
474
#endif
475
#if 6<15
476
# if __ARM_ARCH__>=7
477
	ldr	r2,[r1],#4			@ prefetch
478
# else
479
	ldrb	r2,[r1,#3]
480
# endif
481
	eor	r12,r6,r7			@ a^b, b^c in next round
482
#else
483
	ldr	r2,[sp,#8*4]		@ from future BODY_16_xx
484
	eor	r12,r6,r7			@ a^b, b^c in next round
485
	ldr	r1,[sp,#5*4]	@ from future BODY_16_xx
486
#endif
487
	eor	r0,r0,r6,ror#20	@ Sigma0(a)
488
	and	r3,r3,r12			@ (b^c)&=(a^b)
489
	add	r9,r9,r5			@ d+=h
490
	eor	r3,r3,r7			@ Maj(a,b,c)
491
	add	r5,r5,r0,ror#2	@ h+=Sigma0(a)
492
	@ add	r5,r5,r3			@ h+=Maj(a,b,c)
493
#if __ARM_ARCH__>=7
494
	@ ldr	r2,[r1],#4			@ 7
495
# if 7==15
496
	str	r1,[sp,#17*4]			@ make room for r1
497
# endif
498
	eor	r0,r9,r9,ror#5
499
	add	r5,r5,r3			@ h+=Maj(a,b,c) from the past
500
	eor	r0,r0,r9,ror#19	@ Sigma1(e)
501
# ifndef __ARMEB__
502
	rev	r2,r2
503
# endif
504
#else
505
	@ ldrb	r2,[r1,#3]			@ 7
506
	add	r5,r5,r3			@ h+=Maj(a,b,c) from the past
507
	ldrb	r3,[r1,#2]
508
	ldrb	r0,[r1,#1]
509
	orr	r2,r2,r3,lsl#8
510
	ldrb	r3,[r1],#4
511
	orr	r2,r2,r0,lsl#16
512
# if 7==15
513
	str	r1,[sp,#17*4]			@ make room for r1
514
# endif
515
	eor	r0,r9,r9,ror#5
516
	orr	r2,r2,r3,lsl#24
517
	eor	r0,r0,r9,ror#19	@ Sigma1(e)
518
#endif
519
	ldr	r3,[r14],#4			@ *K256++
520
	add	r4,r4,r2			@ h+=X[i]
521
	str	r2,[sp,#7*4]
522
	eor	r2,r10,r11
523
	add	r4,r4,r0,ror#6	@ h+=Sigma1(e)
524
	and	r2,r2,r9
525
	add	r4,r4,r3			@ h+=K256[i]
526
	eor	r2,r2,r11			@ Ch(e,f,g)
527
	eor	r0,r5,r5,ror#11
528
	add	r4,r4,r2			@ h+=Ch(e,f,g)
529
#if 7==31
530
	and	r3,r3,#0xff
531
	cmp	r3,#0xf2			@ done?
532
#endif
533
#if 7<15
534
# if __ARM_ARCH__>=7
535
	ldr	r2,[r1],#4			@ prefetch
536
# else
537
	ldrb	r2,[r1,#3]
538
# endif
539
	eor	r3,r5,r6			@ a^b, b^c in next round
540
#else
541
	ldr	r2,[sp,#9*4]		@ from future BODY_16_xx
542
	eor	r3,r5,r6			@ a^b, b^c in next round
543
	ldr	r1,[sp,#6*4]	@ from future BODY_16_xx
544
#endif
545
	eor	r0,r0,r5,ror#20	@ Sigma0(a)
546
	and	r12,r12,r3			@ (b^c)&=(a^b)
547
	add	r8,r8,r4			@ d+=h
548
	eor	r12,r12,r6			@ Maj(a,b,c)
549
	add	r4,r4,r0,ror#2	@ h+=Sigma0(a)
550
	@ add	r4,r4,r12			@ h+=Maj(a,b,c)
551
#if __ARM_ARCH__>=7
552
	@ ldr	r2,[r1],#4			@ 8
553
# if 8==15
554
	str	r1,[sp,#17*4]			@ make room for r1
555
# endif
556
	eor	r0,r8,r8,ror#5
557
	add	r4,r4,r12			@ h+=Maj(a,b,c) from the past
558
	eor	r0,r0,r8,ror#19	@ Sigma1(e)
559
# ifndef __ARMEB__
560
	rev	r2,r2
561
# endif
562
#else
563
	@ ldrb	r2,[r1,#3]			@ 8
564
	add	r4,r4,r12			@ h+=Maj(a,b,c) from the past
565
	ldrb	r12,[r1,#2]
566
	ldrb	r0,[r1,#1]
567
	orr	r2,r2,r12,lsl#8
568
	ldrb	r12,[r1],#4
569
	orr	r2,r2,r0,lsl#16
570
# if 8==15
571
	str	r1,[sp,#17*4]			@ make room for r1
572
# endif
573
	eor	r0,r8,r8,ror#5
574
	orr	r2,r2,r12,lsl#24
575
	eor	r0,r0,r8,ror#19	@ Sigma1(e)
576
#endif
577
	ldr	r12,[r14],#4			@ *K256++
578
	add	r11,r11,r2			@ h+=X[i]
579
	str	r2,[sp,#8*4]
580
	eor	r2,r9,r10
581
	add	r11,r11,r0,ror#6	@ h+=Sigma1(e)
582
	and	r2,r2,r8
583
	add	r11,r11,r12			@ h+=K256[i]
584
	eor	r2,r2,r10			@ Ch(e,f,g)
585
	eor	r0,r4,r4,ror#11
586
	add	r11,r11,r2			@ h+=Ch(e,f,g)
587
#if 8==31
588
	and	r12,r12,#0xff
589
	cmp	r12,#0xf2			@ done?
590
#endif
591
#if 8<15
592
# if __ARM_ARCH__>=7
593
	ldr	r2,[r1],#4			@ prefetch
594
# else
595
	ldrb	r2,[r1,#3]
596
# endif
597
	eor	r12,r4,r5			@ a^b, b^c in next round
598
#else
599
	ldr	r2,[sp,#10*4]		@ from future BODY_16_xx
600
	eor	r12,r4,r5			@ a^b, b^c in next round
601
	ldr	r1,[sp,#7*4]	@ from future BODY_16_xx
602
#endif
603
	eor	r0,r0,r4,ror#20	@ Sigma0(a)
604
	and	r3,r3,r12			@ (b^c)&=(a^b)
605
	add	r7,r7,r11			@ d+=h
606
	eor	r3,r3,r5			@ Maj(a,b,c)
607
	add	r11,r11,r0,ror#2	@ h+=Sigma0(a)
608
	@ add	r11,r11,r3			@ h+=Maj(a,b,c)
609
#if __ARM_ARCH__>=7
610
	@ ldr	r2,[r1],#4			@ 9
611
# if 9==15
612
	str	r1,[sp,#17*4]			@ make room for r1
613
# endif
614
	eor	r0,r7,r7,ror#5
615
	add	r11,r11,r3			@ h+=Maj(a,b,c) from the past
616
	eor	r0,r0,r7,ror#19	@ Sigma1(e)
617
# ifndef __ARMEB__
618
	rev	r2,r2
619
# endif
620
#else
621
	@ ldrb	r2,[r1,#3]			@ 9
622
	add	r11,r11,r3			@ h+=Maj(a,b,c) from the past
623
	ldrb	r3,[r1,#2]
624
	ldrb	r0,[r1,#1]
625
	orr	r2,r2,r3,lsl#8
626
	ldrb	r3,[r1],#4
627
	orr	r2,r2,r0,lsl#16
628
# if 9==15
629
	str	r1,[sp,#17*4]			@ make room for r1
630
# endif
631
	eor	r0,r7,r7,ror#5
632
	orr	r2,r2,r3,lsl#24
633
	eor	r0,r0,r7,ror#19	@ Sigma1(e)
634
#endif
635
	ldr	r3,[r14],#4			@ *K256++
636
	add	r10,r10,r2			@ h+=X[i]
637
	str	r2,[sp,#9*4]
638
	eor	r2,r8,r9
639
	add	r10,r10,r0,ror#6	@ h+=Sigma1(e)
640
	and	r2,r2,r7
641
	add	r10,r10,r3			@ h+=K256[i]
642
	eor	r2,r2,r9			@ Ch(e,f,g)
643
	eor	r0,r11,r11,ror#11
644
	add	r10,r10,r2			@ h+=Ch(e,f,g)
645
#if 9==31
646
	and	r3,r3,#0xff
647
	cmp	r3,#0xf2			@ done?
648
#endif
649
#if 9<15
650
# if __ARM_ARCH__>=7
651
	ldr	r2,[r1],#4			@ prefetch
652
# else
653
	ldrb	r2,[r1,#3]
654
# endif
655
	eor	r3,r11,r4			@ a^b, b^c in next round
656
#else
657
	ldr	r2,[sp,#11*4]		@ from future BODY_16_xx
658
	eor	r3,r11,r4			@ a^b, b^c in next round
659
	ldr	r1,[sp,#8*4]	@ from future BODY_16_xx
660
#endif
661
	eor	r0,r0,r11,ror#20	@ Sigma0(a)
662
	and	r12,r12,r3			@ (b^c)&=(a^b)
663
	add	r6,r6,r10			@ d+=h
664
	eor	r12,r12,r4			@ Maj(a,b,c)
665
	add	r10,r10,r0,ror#2	@ h+=Sigma0(a)
666
	@ add	r10,r10,r12			@ h+=Maj(a,b,c)
667
#if __ARM_ARCH__>=7
668
	@ ldr	r2,[r1],#4			@ 10
669
# if 10==15
670
	str	r1,[sp,#17*4]			@ make room for r1
671
# endif
672
	eor	r0,r6,r6,ror#5
673
	add	r10,r10,r12			@ h+=Maj(a,b,c) from the past
674
	eor	r0,r0,r6,ror#19	@ Sigma1(e)
675
# ifndef __ARMEB__
676
	rev	r2,r2
677
# endif
678
#else
679
	@ ldrb	r2,[r1,#3]			@ 10
680
	add	r10,r10,r12			@ h+=Maj(a,b,c) from the past
681
	ldrb	r12,[r1,#2]
682
	ldrb	r0,[r1,#1]
683
	orr	r2,r2,r12,lsl#8
684
	ldrb	r12,[r1],#4
685
	orr	r2,r2,r0,lsl#16
686
# if 10==15
687
	str	r1,[sp,#17*4]			@ make room for r1
688
# endif
689
	eor	r0,r6,r6,ror#5
690
	orr	r2,r2,r12,lsl#24
691
	eor	r0,r0,r6,ror#19	@ Sigma1(e)
692
#endif
693
	ldr	r12,[r14],#4			@ *K256++
694
	add	r9,r9,r2			@ h+=X[i]
695
	str	r2,[sp,#10*4]
696
	eor	r2,r7,r8
697
	add	r9,r9,r0,ror#6	@ h+=Sigma1(e)
698
	and	r2,r2,r6
699
	add	r9,r9,r12			@ h+=K256[i]
700
	eor	r2,r2,r8			@ Ch(e,f,g)
701
	eor	r0,r10,r10,ror#11
702
	add	r9,r9,r2			@ h+=Ch(e,f,g)
703
#if 10==31
704
	and	r12,r12,#0xff
705
	cmp	r12,#0xf2			@ done?
706
#endif
707
#if 10<15
708
# if __ARM_ARCH__>=7
709
	ldr	r2,[r1],#4			@ prefetch
710
# else
711
	ldrb	r2,[r1,#3]
712
# endif
713
	eor	r12,r10,r11			@ a^b, b^c in next round
714
#else
715
	ldr	r2,[sp,#12*4]		@ from future BODY_16_xx
716
	eor	r12,r10,r11			@ a^b, b^c in next round
717
	ldr	r1,[sp,#9*4]	@ from future BODY_16_xx
718
#endif
719
	eor	r0,r0,r10,ror#20	@ Sigma0(a)
720
	and	r3,r3,r12			@ (b^c)&=(a^b)
721
	add	r5,r5,r9			@ d+=h
722
	eor	r3,r3,r11			@ Maj(a,b,c)
723
	add	r9,r9,r0,ror#2	@ h+=Sigma0(a)
724
	@ add	r9,r9,r3			@ h+=Maj(a,b,c)
725
#if __ARM_ARCH__>=7
726
	@ ldr	r2,[r1],#4			@ 11
727
# if 11==15
728
	str	r1,[sp,#17*4]			@ make room for r1
729
# endif
730
	eor	r0,r5,r5,ror#5
731
	add	r9,r9,r3			@ h+=Maj(a,b,c) from the past
732
	eor	r0,r0,r5,ror#19	@ Sigma1(e)
733
# ifndef __ARMEB__
734
	rev	r2,r2
735
# endif
736
#else
737
	@ ldrb	r2,[r1,#3]			@ 11
738
	add	r9,r9,r3			@ h+=Maj(a,b,c) from the past
739
	ldrb	r3,[r1,#2]
740
	ldrb	r0,[r1,#1]
741
	orr	r2,r2,r3,lsl#8
742
	ldrb	r3,[r1],#4
743
	orr	r2,r2,r0,lsl#16
744
# if 11==15
745
	str	r1,[sp,#17*4]			@ make room for r1
746
# endif
747
	eor	r0,r5,r5,ror#5
748
	orr	r2,r2,r3,lsl#24
749
	eor	r0,r0,r5,ror#19	@ Sigma1(e)
750
#endif
751
	ldr	r3,[r14],#4			@ *K256++
752
	add	r8,r8,r2			@ h+=X[i]
753
	str	r2,[sp,#11*4]
754
	eor	r2,r6,r7
755
	add	r8,r8,r0,ror#6	@ h+=Sigma1(e)
756
	and	r2,r2,r5
757
	add	r8,r8,r3			@ h+=K256[i]
758
	eor	r2,r2,r7			@ Ch(e,f,g)
759
	eor	r0,r9,r9,ror#11
760
	add	r8,r8,r2			@ h+=Ch(e,f,g)
761
#if 11==31
762
	and	r3,r3,#0xff
763
	cmp	r3,#0xf2			@ done?
764
#endif
765
#if 11<15
766
# if __ARM_ARCH__>=7
767
	ldr	r2,[r1],#4			@ prefetch
768
# else
769
	ldrb	r2,[r1,#3]
770
# endif
771
	eor	r3,r9,r10			@ a^b, b^c in next round
772
#else
773
	ldr	r2,[sp,#13*4]		@ from future BODY_16_xx
774
	eor	r3,r9,r10			@ a^b, b^c in next round
775
	ldr	r1,[sp,#10*4]	@ from future BODY_16_xx
776
#endif
777
	eor	r0,r0,r9,ror#20	@ Sigma0(a)
778
	and	r12,r12,r3			@ (b^c)&=(a^b)
779
	add	r4,r4,r8			@ d+=h
780
	eor	r12,r12,r10			@ Maj(a,b,c)
781
	add	r8,r8,r0,ror#2	@ h+=Sigma0(a)
782
	@ add	r8,r8,r12			@ h+=Maj(a,b,c)
783
#if __ARM_ARCH__>=7
784
	@ ldr	r2,[r1],#4			@ 12
785
# if 12==15
786
	str	r1,[sp,#17*4]			@ make room for r1
787
# endif
788
	eor	r0,r4,r4,ror#5
789
	add	r8,r8,r12			@ h+=Maj(a,b,c) from the past
790
	eor	r0,r0,r4,ror#19	@ Sigma1(e)
791
# ifndef __ARMEB__
792
	rev	r2,r2
793
# endif
794
#else
795
	@ ldrb	r2,[r1,#3]			@ 12
796
	add	r8,r8,r12			@ h+=Maj(a,b,c) from the past
797
	ldrb	r12,[r1,#2]
798
	ldrb	r0,[r1,#1]
799
	orr	r2,r2,r12,lsl#8
800
	ldrb	r12,[r1],#4
801
	orr	r2,r2,r0,lsl#16
802
# if 12==15
803
	str	r1,[sp,#17*4]			@ make room for r1
804
# endif
805
	eor	r0,r4,r4,ror#5
806
	orr	r2,r2,r12,lsl#24
807
	eor	r0,r0,r4,ror#19	@ Sigma1(e)
808
#endif
809
	ldr	r12,[r14],#4			@ *K256++
810
	add	r7,r7,r2			@ h+=X[i]
811
	str	r2,[sp,#12*4]
812
	eor	r2,r5,r6
813
	add	r7,r7,r0,ror#6	@ h+=Sigma1(e)
814
	and	r2,r2,r4
815
	add	r7,r7,r12			@ h+=K256[i]
816
	eor	r2,r2,r6			@ Ch(e,f,g)
817
	eor	r0,r8,r8,ror#11
818
	add	r7,r7,r2			@ h+=Ch(e,f,g)
819
#if 12==31
820
	and	r12,r12,#0xff
821
	cmp	r12,#0xf2			@ done?
822
#endif
823
#if 12<15
824
# if __ARM_ARCH__>=7
825
	ldr	r2,[r1],#4			@ prefetch
826
# else
827
	ldrb	r2,[r1,#3]
828
# endif
829
	eor	r12,r8,r9			@ a^b, b^c in next round
830
#else
831
	ldr	r2,[sp,#14*4]		@ from future BODY_16_xx
832
	eor	r12,r8,r9			@ a^b, b^c in next round
833
	ldr	r1,[sp,#11*4]	@ from future BODY_16_xx
834
#endif
835
	eor	r0,r0,r8,ror#20	@ Sigma0(a)
836
	and	r3,r3,r12			@ (b^c)&=(a^b)
837
	add	r11,r11,r7			@ d+=h
838
	eor	r3,r3,r9			@ Maj(a,b,c)
839
	add	r7,r7,r0,ror#2	@ h+=Sigma0(a)
840
	@ add	r7,r7,r3			@ h+=Maj(a,b,c)
841
#if __ARM_ARCH__>=7
842
	@ ldr	r2,[r1],#4			@ 13
843
# if 13==15
844
	str	r1,[sp,#17*4]			@ make room for r1
845
# endif
846
	eor	r0,r11,r11,ror#5
847
	add	r7,r7,r3			@ h+=Maj(a,b,c) from the past
848
	eor	r0,r0,r11,ror#19	@ Sigma1(e)
849
# ifndef __ARMEB__
850
	rev	r2,r2
851
# endif
852
#else
853
	@ ldrb	r2,[r1,#3]			@ 13
854
	add	r7,r7,r3			@ h+=Maj(a,b,c) from the past
855
	ldrb	r3,[r1,#2]
856
	ldrb	r0,[r1,#1]
857
	orr	r2,r2,r3,lsl#8
858
	ldrb	r3,[r1],#4
859
	orr	r2,r2,r0,lsl#16
860
# if 13==15
861
	str	r1,[sp,#17*4]			@ make room for r1
862
# endif
863
	eor	r0,r11,r11,ror#5
864
	orr	r2,r2,r3,lsl#24
865
	eor	r0,r0,r11,ror#19	@ Sigma1(e)
866
#endif
867
	ldr	r3,[r14],#4			@ *K256++
868
	add	r6,r6,r2			@ h+=X[i]
869
	str	r2,[sp,#13*4]
870
	eor	r2,r4,r5
871
	add	r6,r6,r0,ror#6	@ h+=Sigma1(e)
872
	and	r2,r2,r11
873
	add	r6,r6,r3			@ h+=K256[i]
874
	eor	r2,r2,r5			@ Ch(e,f,g)
875
	eor	r0,r7,r7,ror#11
876
	add	r6,r6,r2			@ h+=Ch(e,f,g)
877
#if 13==31
878
	and	r3,r3,#0xff
879
	cmp	r3,#0xf2			@ done?
880
#endif
881
#if 13<15
882
# if __ARM_ARCH__>=7
883
	ldr	r2,[r1],#4			@ prefetch
884
# else
885
	ldrb	r2,[r1,#3]
886
# endif
887
	eor	r3,r7,r8			@ a^b, b^c in next round
888
#else
889
	ldr	r2,[sp,#15*4]		@ from future BODY_16_xx
890
	eor	r3,r7,r8			@ a^b, b^c in next round
891
	ldr	r1,[sp,#12*4]	@ from future BODY_16_xx
892
#endif
893
	eor	r0,r0,r7,ror#20	@ Sigma0(a)
894
	and	r12,r12,r3			@ (b^c)&=(a^b)
895
	add	r10,r10,r6			@ d+=h
896
	eor	r12,r12,r8			@ Maj(a,b,c)
897
	add	r6,r6,r0,ror#2	@ h+=Sigma0(a)
898
	@ add	r6,r6,r12			@ h+=Maj(a,b,c)
899
#if __ARM_ARCH__>=7
900
	@ ldr	r2,[r1],#4			@ 14
901
# if 14==15
902
	str	r1,[sp,#17*4]			@ make room for r1
903
# endif
904
	eor	r0,r10,r10,ror#5
905
	add	r6,r6,r12			@ h+=Maj(a,b,c) from the past
906
	eor	r0,r0,r10,ror#19	@ Sigma1(e)
907
# ifndef __ARMEB__
908
	rev	r2,r2
909
# endif
910
#else
911
	@ ldrb	r2,[r1,#3]			@ 14
912
	add	r6,r6,r12			@ h+=Maj(a,b,c) from the past
913
	ldrb	r12,[r1,#2]
914
	ldrb	r0,[r1,#1]
915
	orr	r2,r2,r12,lsl#8
916
	ldrb	r12,[r1],#4
917
	orr	r2,r2,r0,lsl#16
918
# if 14==15
919
	str	r1,[sp,#17*4]			@ make room for r1
920
# endif
921
	eor	r0,r10,r10,ror#5
922
	orr	r2,r2,r12,lsl#24
923
	eor	r0,r0,r10,ror#19	@ Sigma1(e)
924
#endif
925
	ldr	r12,[r14],#4			@ *K256++
926
	add	r5,r5,r2			@ h+=X[i]
927
	str	r2,[sp,#14*4]
928
	eor	r2,r11,r4
929
	add	r5,r5,r0,ror#6	@ h+=Sigma1(e)
930
	and	r2,r2,r10
931
	add	r5,r5,r12			@ h+=K256[i]
932
	eor	r2,r2,r4			@ Ch(e,f,g)
933
	eor	r0,r6,r6,ror#11
934
	add	r5,r5,r2			@ h+=Ch(e,f,g)
935
#if 14==31
936
	and	r12,r12,#0xff
937
	cmp	r12,#0xf2			@ done?
938
#endif
939
#if 14<15
940
# if __ARM_ARCH__>=7
941
	ldr	r2,[r1],#4			@ prefetch
942
# else
943
	ldrb	r2,[r1,#3]
944
# endif
945
	eor	r12,r6,r7			@ a^b, b^c in next round
946
#else
947
	ldr	r2,[sp,#0*4]		@ from future BODY_16_xx
948
	eor	r12,r6,r7			@ a^b, b^c in next round
949
	ldr	r1,[sp,#13*4]	@ from future BODY_16_xx
950
#endif
951
	eor	r0,r0,r6,ror#20	@ Sigma0(a)
952
	and	r3,r3,r12			@ (b^c)&=(a^b)
953
	add	r9,r9,r5			@ d+=h
954
	eor	r3,r3,r7			@ Maj(a,b,c)
955
	add	r5,r5,r0,ror#2	@ h+=Sigma0(a)
956
	@ add	r5,r5,r3			@ h+=Maj(a,b,c)
957
#if __ARM_ARCH__>=7
958
	@ ldr	r2,[r1],#4			@ 15
959
# if 15==15
960
	str	r1,[sp,#17*4]			@ make room for r1
961
# endif
962
	eor	r0,r9,r9,ror#5
963
	add	r5,r5,r3			@ h+=Maj(a,b,c) from the past
964
	eor	r0,r0,r9,ror#19	@ Sigma1(e)
965
# ifndef __ARMEB__
966
	rev	r2,r2
967
# endif
968
#else
969
	@ ldrb	r2,[r1,#3]			@ 15
970
	add	r5,r5,r3			@ h+=Maj(a,b,c) from the past
971
	ldrb	r3,[r1,#2]
972
	ldrb	r0,[r1,#1]
973
	orr	r2,r2,r3,lsl#8
974
	ldrb	r3,[r1],#4
975
	orr	r2,r2,r0,lsl#16
976
# if 15==15
977
	str	r1,[sp,#17*4]			@ make room for r1
978
# endif
979
	eor	r0,r9,r9,ror#5
980
	orr	r2,r2,r3,lsl#24
981
	eor	r0,r0,r9,ror#19	@ Sigma1(e)
982
#endif
983
	ldr	r3,[r14],#4			@ *K256++
984
	add	r4,r4,r2			@ h+=X[i]
985
	str	r2,[sp,#15*4]
986
	eor	r2,r10,r11
987
	add	r4,r4,r0,ror#6	@ h+=Sigma1(e)
988
	and	r2,r2,r9
989
	add	r4,r4,r3			@ h+=K256[i]
990
	eor	r2,r2,r11			@ Ch(e,f,g)
991
	eor	r0,r5,r5,ror#11
992
	add	r4,r4,r2			@ h+=Ch(e,f,g)
993
#if 15==31
994
	and	r3,r3,#0xff
995
	cmp	r3,#0xf2			@ done?
996
#endif
997
#if 15<15
998
# if __ARM_ARCH__>=7
999
	ldr	r2,[r1],#4			@ prefetch
1000
# else
1001
	ldrb	r2,[r1,#3]
1002
# endif
1003
	eor	r3,r5,r6			@ a^b, b^c in next round
1004
#else
1005
	ldr	r2,[sp,#1*4]		@ from future BODY_16_xx
1006
	eor	r3,r5,r6			@ a^b, b^c in next round
1007
	ldr	r1,[sp,#14*4]	@ from future BODY_16_xx
1008
#endif
1009
	eor	r0,r0,r5,ror#20	@ Sigma0(a)
1010
	and	r12,r12,r3			@ (b^c)&=(a^b)
1011
	add	r8,r8,r4			@ d+=h
1012
	eor	r12,r12,r6			@ Maj(a,b,c)
1013
	add	r4,r4,r0,ror#2	@ h+=Sigma0(a)
1014
	@ add	r4,r4,r12			@ h+=Maj(a,b,c)
1015
.Lrounds_16_xx:
1016
	@ ldr	r2,[sp,#1*4]		@ 16
1017
	@ ldr	r1,[sp,#14*4]
1018
	mov	r0,r2,ror#7
1019
	add	r4,r4,r12			@ h+=Maj(a,b,c) from the past
1020
	mov	r12,r1,ror#17
1021
	eor	r0,r0,r2,ror#18
1022
	eor	r12,r12,r1,ror#19
1023
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1024
	ldr	r2,[sp,#0*4]
1025
	eor	r12,r12,r1,lsr#10	@ sigma1(X[i+14])
1026
	ldr	r1,[sp,#9*4]
1027

1028
	add	r12,r12,r0
1029
	eor	r0,r8,r8,ror#5	@ from BODY_00_15
1030
	add	r2,r2,r12
1031
	eor	r0,r0,r8,ror#19	@ Sigma1(e)
1032
	add	r2,r2,r1			@ X[i]
1033
	ldr	r12,[r14],#4			@ *K256++
1034
	add	r11,r11,r2			@ h+=X[i]
1035
	str	r2,[sp,#0*4]
1036
	eor	r2,r9,r10
1037
	add	r11,r11,r0,ror#6	@ h+=Sigma1(e)
1038
	and	r2,r2,r8
1039
	add	r11,r11,r12			@ h+=K256[i]
1040
	eor	r2,r2,r10			@ Ch(e,f,g)
1041
	eor	r0,r4,r4,ror#11
1042
	add	r11,r11,r2			@ h+=Ch(e,f,g)
1043
#if 16==31
1044
	and	r12,r12,#0xff
1045
	cmp	r12,#0xf2			@ done?
1046
#endif
1047
#if 16<15
1048
# if __ARM_ARCH__>=7
1049
	ldr	r2,[r1],#4			@ prefetch
1050
# else
1051
	ldrb	r2,[r1,#3]
1052
# endif
1053
	eor	r12,r4,r5			@ a^b, b^c in next round
1054
#else
1055
	ldr	r2,[sp,#2*4]		@ from future BODY_16_xx
1056
	eor	r12,r4,r5			@ a^b, b^c in next round
1057
	ldr	r1,[sp,#15*4]	@ from future BODY_16_xx
1058
#endif
1059
	eor	r0,r0,r4,ror#20	@ Sigma0(a)
1060
	and	r3,r3,r12			@ (b^c)&=(a^b)
1061
	add	r7,r7,r11			@ d+=h
1062
	eor	r3,r3,r5			@ Maj(a,b,c)
1063
	add	r11,r11,r0,ror#2	@ h+=Sigma0(a)
1064
	@ add	r11,r11,r3			@ h+=Maj(a,b,c)
1065
	@ ldr	r2,[sp,#2*4]		@ 17
1066
	@ ldr	r1,[sp,#15*4]
1067
	mov	r0,r2,ror#7
1068
	add	r11,r11,r3			@ h+=Maj(a,b,c) from the past
1069
	mov	r3,r1,ror#17
1070
	eor	r0,r0,r2,ror#18
1071
	eor	r3,r3,r1,ror#19
1072
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1073
	ldr	r2,[sp,#1*4]
1074
	eor	r3,r3,r1,lsr#10	@ sigma1(X[i+14])
1075
	ldr	r1,[sp,#10*4]
1076

1077
	add	r3,r3,r0
1078
	eor	r0,r7,r7,ror#5	@ from BODY_00_15
1079
	add	r2,r2,r3
1080
	eor	r0,r0,r7,ror#19	@ Sigma1(e)
1081
	add	r2,r2,r1			@ X[i]
1082
	ldr	r3,[r14],#4			@ *K256++
1083
	add	r10,r10,r2			@ h+=X[i]
1084
	str	r2,[sp,#1*4]
1085
	eor	r2,r8,r9
1086
	add	r10,r10,r0,ror#6	@ h+=Sigma1(e)
1087
	and	r2,r2,r7
1088
	add	r10,r10,r3			@ h+=K256[i]
1089
	eor	r2,r2,r9			@ Ch(e,f,g)
1090
	eor	r0,r11,r11,ror#11
1091
	add	r10,r10,r2			@ h+=Ch(e,f,g)
1092
#if 17==31
1093
	and	r3,r3,#0xff
1094
	cmp	r3,#0xf2			@ done?
1095
#endif
1096
#if 17<15
1097
# if __ARM_ARCH__>=7
1098
	ldr	r2,[r1],#4			@ prefetch
1099
# else
1100
	ldrb	r2,[r1,#3]
1101
# endif
1102
	eor	r3,r11,r4			@ a^b, b^c in next round
1103
#else
1104
	ldr	r2,[sp,#3*4]		@ from future BODY_16_xx
1105
	eor	r3,r11,r4			@ a^b, b^c in next round
1106
	ldr	r1,[sp,#0*4]	@ from future BODY_16_xx
1107
#endif
1108
	eor	r0,r0,r11,ror#20	@ Sigma0(a)
1109
	and	r12,r12,r3			@ (b^c)&=(a^b)
1110
	add	r6,r6,r10			@ d+=h
1111
	eor	r12,r12,r4			@ Maj(a,b,c)
1112
	add	r10,r10,r0,ror#2	@ h+=Sigma0(a)
1113
	@ add	r10,r10,r12			@ h+=Maj(a,b,c)
1114
	@ ldr	r2,[sp,#3*4]		@ 18
1115
	@ ldr	r1,[sp,#0*4]
1116
	mov	r0,r2,ror#7
1117
	add	r10,r10,r12			@ h+=Maj(a,b,c) from the past
1118
	mov	r12,r1,ror#17
1119
	eor	r0,r0,r2,ror#18
1120
	eor	r12,r12,r1,ror#19
1121
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1122
	ldr	r2,[sp,#2*4]
1123
	eor	r12,r12,r1,lsr#10	@ sigma1(X[i+14])
1124
	ldr	r1,[sp,#11*4]
1125

1126
	add	r12,r12,r0
1127
	eor	r0,r6,r6,ror#5	@ from BODY_00_15
1128
	add	r2,r2,r12
1129
	eor	r0,r0,r6,ror#19	@ Sigma1(e)
1130
	add	r2,r2,r1			@ X[i]
1131
	ldr	r12,[r14],#4			@ *K256++
1132
	add	r9,r9,r2			@ h+=X[i]
1133
	str	r2,[sp,#2*4]
1134
	eor	r2,r7,r8
1135
	add	r9,r9,r0,ror#6	@ h+=Sigma1(e)
1136
	and	r2,r2,r6
1137
	add	r9,r9,r12			@ h+=K256[i]
1138
	eor	r2,r2,r8			@ Ch(e,f,g)
1139
	eor	r0,r10,r10,ror#11
1140
	add	r9,r9,r2			@ h+=Ch(e,f,g)
1141
#if 18==31
1142
	and	r12,r12,#0xff
1143
	cmp	r12,#0xf2			@ done?
1144
#endif
1145
#if 18<15
1146
# if __ARM_ARCH__>=7
1147
	ldr	r2,[r1],#4			@ prefetch
1148
# else
1149
	ldrb	r2,[r1,#3]
1150
# endif
1151
	eor	r12,r10,r11			@ a^b, b^c in next round
1152
#else
1153
	ldr	r2,[sp,#4*4]		@ from future BODY_16_xx
1154
	eor	r12,r10,r11			@ a^b, b^c in next round
1155
	ldr	r1,[sp,#1*4]	@ from future BODY_16_xx
1156
#endif
1157
	eor	r0,r0,r10,ror#20	@ Sigma0(a)
1158
	and	r3,r3,r12			@ (b^c)&=(a^b)
1159
	add	r5,r5,r9			@ d+=h
1160
	eor	r3,r3,r11			@ Maj(a,b,c)
1161
	add	r9,r9,r0,ror#2	@ h+=Sigma0(a)
1162
	@ add	r9,r9,r3			@ h+=Maj(a,b,c)
1163
	@ ldr	r2,[sp,#4*4]		@ 19
1164
	@ ldr	r1,[sp,#1*4]
1165
	mov	r0,r2,ror#7
1166
	add	r9,r9,r3			@ h+=Maj(a,b,c) from the past
1167
	mov	r3,r1,ror#17
1168
	eor	r0,r0,r2,ror#18
1169
	eor	r3,r3,r1,ror#19
1170
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1171
	ldr	r2,[sp,#3*4]
1172
	eor	r3,r3,r1,lsr#10	@ sigma1(X[i+14])
1173
	ldr	r1,[sp,#12*4]
1174

1175
	add	r3,r3,r0
1176
	eor	r0,r5,r5,ror#5	@ from BODY_00_15
1177
	add	r2,r2,r3
1178
	eor	r0,r0,r5,ror#19	@ Sigma1(e)
1179
	add	r2,r2,r1			@ X[i]
1180
	ldr	r3,[r14],#4			@ *K256++
1181
	add	r8,r8,r2			@ h+=X[i]
1182
	str	r2,[sp,#3*4]
1183
	eor	r2,r6,r7
1184
	add	r8,r8,r0,ror#6	@ h+=Sigma1(e)
1185
	and	r2,r2,r5
1186
	add	r8,r8,r3			@ h+=K256[i]
1187
	eor	r2,r2,r7			@ Ch(e,f,g)
1188
	eor	r0,r9,r9,ror#11
1189
	add	r8,r8,r2			@ h+=Ch(e,f,g)
1190
#if 19==31
1191
	and	r3,r3,#0xff
1192
	cmp	r3,#0xf2			@ done?
1193
#endif
1194
#if 19<15
1195
# if __ARM_ARCH__>=7
1196
	ldr	r2,[r1],#4			@ prefetch
1197
# else
1198
	ldrb	r2,[r1,#3]
1199
# endif
1200
	eor	r3,r9,r10			@ a^b, b^c in next round
1201
#else
1202
	ldr	r2,[sp,#5*4]		@ from future BODY_16_xx
1203
	eor	r3,r9,r10			@ a^b, b^c in next round
1204
	ldr	r1,[sp,#2*4]	@ from future BODY_16_xx
1205
#endif
1206
	eor	r0,r0,r9,ror#20	@ Sigma0(a)
1207
	and	r12,r12,r3			@ (b^c)&=(a^b)
1208
	add	r4,r4,r8			@ d+=h
1209
	eor	r12,r12,r10			@ Maj(a,b,c)
1210
	add	r8,r8,r0,ror#2	@ h+=Sigma0(a)
1211
	@ add	r8,r8,r12			@ h+=Maj(a,b,c)
1212
	@ ldr	r2,[sp,#5*4]		@ 20
1213
	@ ldr	r1,[sp,#2*4]
1214
	mov	r0,r2,ror#7
1215
	add	r8,r8,r12			@ h+=Maj(a,b,c) from the past
1216
	mov	r12,r1,ror#17
1217
	eor	r0,r0,r2,ror#18
1218
	eor	r12,r12,r1,ror#19
1219
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1220
	ldr	r2,[sp,#4*4]
1221
	eor	r12,r12,r1,lsr#10	@ sigma1(X[i+14])
1222
	ldr	r1,[sp,#13*4]
1223

1224
	add	r12,r12,r0
1225
	eor	r0,r4,r4,ror#5	@ from BODY_00_15
1226
	add	r2,r2,r12
1227
	eor	r0,r0,r4,ror#19	@ Sigma1(e)
1228
	add	r2,r2,r1			@ X[i]
1229
	ldr	r12,[r14],#4			@ *K256++
1230
	add	r7,r7,r2			@ h+=X[i]
1231
	str	r2,[sp,#4*4]
1232
	eor	r2,r5,r6
1233
	add	r7,r7,r0,ror#6	@ h+=Sigma1(e)
1234
	and	r2,r2,r4
1235
	add	r7,r7,r12			@ h+=K256[i]
1236
	eor	r2,r2,r6			@ Ch(e,f,g)
1237
	eor	r0,r8,r8,ror#11
1238
	add	r7,r7,r2			@ h+=Ch(e,f,g)
1239
#if 20==31
1240
	and	r12,r12,#0xff
1241
	cmp	r12,#0xf2			@ done?
1242
#endif
1243
#if 20<15
1244
# if __ARM_ARCH__>=7
1245
	ldr	r2,[r1],#4			@ prefetch
1246
# else
1247
	ldrb	r2,[r1,#3]
1248
# endif
1249
	eor	r12,r8,r9			@ a^b, b^c in next round
1250
#else
1251
	ldr	r2,[sp,#6*4]		@ from future BODY_16_xx
1252
	eor	r12,r8,r9			@ a^b, b^c in next round
1253
	ldr	r1,[sp,#3*4]	@ from future BODY_16_xx
1254
#endif
1255
	eor	r0,r0,r8,ror#20	@ Sigma0(a)
1256
	and	r3,r3,r12			@ (b^c)&=(a^b)
1257
	add	r11,r11,r7			@ d+=h
1258
	eor	r3,r3,r9			@ Maj(a,b,c)
1259
	add	r7,r7,r0,ror#2	@ h+=Sigma0(a)
1260
	@ add	r7,r7,r3			@ h+=Maj(a,b,c)
1261
	@ ldr	r2,[sp,#6*4]		@ 21
1262
	@ ldr	r1,[sp,#3*4]
1263
	mov	r0,r2,ror#7
1264
	add	r7,r7,r3			@ h+=Maj(a,b,c) from the past
1265
	mov	r3,r1,ror#17
1266
	eor	r0,r0,r2,ror#18
1267
	eor	r3,r3,r1,ror#19
1268
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1269
	ldr	r2,[sp,#5*4]
1270
	eor	r3,r3,r1,lsr#10	@ sigma1(X[i+14])
1271
	ldr	r1,[sp,#14*4]
1272

1273
	add	r3,r3,r0
1274
	eor	r0,r11,r11,ror#5	@ from BODY_00_15
1275
	add	r2,r2,r3
1276
	eor	r0,r0,r11,ror#19	@ Sigma1(e)
1277
	add	r2,r2,r1			@ X[i]
1278
	ldr	r3,[r14],#4			@ *K256++
1279
	add	r6,r6,r2			@ h+=X[i]
1280
	str	r2,[sp,#5*4]
1281
	eor	r2,r4,r5
1282
	add	r6,r6,r0,ror#6	@ h+=Sigma1(e)
1283
	and	r2,r2,r11
1284
	add	r6,r6,r3			@ h+=K256[i]
1285
	eor	r2,r2,r5			@ Ch(e,f,g)
1286
	eor	r0,r7,r7,ror#11
1287
	add	r6,r6,r2			@ h+=Ch(e,f,g)
1288
#if 21==31
1289
	and	r3,r3,#0xff
1290
	cmp	r3,#0xf2			@ done?
1291
#endif
1292
#if 21<15
1293
# if __ARM_ARCH__>=7
1294
	ldr	r2,[r1],#4			@ prefetch
1295
# else
1296
	ldrb	r2,[r1,#3]
1297
# endif
1298
	eor	r3,r7,r8			@ a^b, b^c in next round
1299
#else
1300
	ldr	r2,[sp,#7*4]		@ from future BODY_16_xx
1301
	eor	r3,r7,r8			@ a^b, b^c in next round
1302
	ldr	r1,[sp,#4*4]	@ from future BODY_16_xx
1303
#endif
1304
	eor	r0,r0,r7,ror#20	@ Sigma0(a)
1305
	and	r12,r12,r3			@ (b^c)&=(a^b)
1306
	add	r10,r10,r6			@ d+=h
1307
	eor	r12,r12,r8			@ Maj(a,b,c)
1308
	add	r6,r6,r0,ror#2	@ h+=Sigma0(a)
1309
	@ add	r6,r6,r12			@ h+=Maj(a,b,c)
1310
	@ ldr	r2,[sp,#7*4]		@ 22
1311
	@ ldr	r1,[sp,#4*4]
1312
	mov	r0,r2,ror#7
1313
	add	r6,r6,r12			@ h+=Maj(a,b,c) from the past
1314
	mov	r12,r1,ror#17
1315
	eor	r0,r0,r2,ror#18
1316
	eor	r12,r12,r1,ror#19
1317
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1318
	ldr	r2,[sp,#6*4]
1319
	eor	r12,r12,r1,lsr#10	@ sigma1(X[i+14])
1320
	ldr	r1,[sp,#15*4]
1321

1322
	add	r12,r12,r0
1323
	eor	r0,r10,r10,ror#5	@ from BODY_00_15
1324
	add	r2,r2,r12
1325
	eor	r0,r0,r10,ror#19	@ Sigma1(e)
1326
	add	r2,r2,r1			@ X[i]
1327
	ldr	r12,[r14],#4			@ *K256++
1328
	add	r5,r5,r2			@ h+=X[i]
1329
	str	r2,[sp,#6*4]
1330
	eor	r2,r11,r4
1331
	add	r5,r5,r0,ror#6	@ h+=Sigma1(e)
1332
	and	r2,r2,r10
1333
	add	r5,r5,r12			@ h+=K256[i]
1334
	eor	r2,r2,r4			@ Ch(e,f,g)
1335
	eor	r0,r6,r6,ror#11
1336
	add	r5,r5,r2			@ h+=Ch(e,f,g)
1337
#if 22==31
1338
	and	r12,r12,#0xff
1339
	cmp	r12,#0xf2			@ done?
1340
#endif
1341
#if 22<15
1342
# if __ARM_ARCH__>=7
1343
	ldr	r2,[r1],#4			@ prefetch
1344
# else
1345
	ldrb	r2,[r1,#3]
1346
# endif
1347
	eor	r12,r6,r7			@ a^b, b^c in next round
1348
#else
1349
	ldr	r2,[sp,#8*4]		@ from future BODY_16_xx
1350
	eor	r12,r6,r7			@ a^b, b^c in next round
1351
	ldr	r1,[sp,#5*4]	@ from future BODY_16_xx
1352
#endif
1353
	eor	r0,r0,r6,ror#20	@ Sigma0(a)
1354
	and	r3,r3,r12			@ (b^c)&=(a^b)
1355
	add	r9,r9,r5			@ d+=h
1356
	eor	r3,r3,r7			@ Maj(a,b,c)
1357
	add	r5,r5,r0,ror#2	@ h+=Sigma0(a)
1358
	@ add	r5,r5,r3			@ h+=Maj(a,b,c)
1359
	@ ldr	r2,[sp,#8*4]		@ 23
1360
	@ ldr	r1,[sp,#5*4]
1361
	mov	r0,r2,ror#7
1362
	add	r5,r5,r3			@ h+=Maj(a,b,c) from the past
1363
	mov	r3,r1,ror#17
1364
	eor	r0,r0,r2,ror#18
1365
	eor	r3,r3,r1,ror#19
1366
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1367
	ldr	r2,[sp,#7*4]
1368
	eor	r3,r3,r1,lsr#10	@ sigma1(X[i+14])
1369
	ldr	r1,[sp,#0*4]
1370

1371
	add	r3,r3,r0
1372
	eor	r0,r9,r9,ror#5	@ from BODY_00_15
1373
	add	r2,r2,r3
1374
	eor	r0,r0,r9,ror#19	@ Sigma1(e)
1375
	add	r2,r2,r1			@ X[i]
1376
	ldr	r3,[r14],#4			@ *K256++
1377
	add	r4,r4,r2			@ h+=X[i]
1378
	str	r2,[sp,#7*4]
1379
	eor	r2,r10,r11
1380
	add	r4,r4,r0,ror#6	@ h+=Sigma1(e)
1381
	and	r2,r2,r9
1382
	add	r4,r4,r3			@ h+=K256[i]
1383
	eor	r2,r2,r11			@ Ch(e,f,g)
1384
	eor	r0,r5,r5,ror#11
1385
	add	r4,r4,r2			@ h+=Ch(e,f,g)
1386
#if 23==31
1387
	and	r3,r3,#0xff
1388
	cmp	r3,#0xf2			@ done?
1389
#endif
1390
#if 23<15
1391
# if __ARM_ARCH__>=7
1392
	ldr	r2,[r1],#4			@ prefetch
1393
# else
1394
	ldrb	r2,[r1,#3]
1395
# endif
1396
	eor	r3,r5,r6			@ a^b, b^c in next round
1397
#else
1398
	ldr	r2,[sp,#9*4]		@ from future BODY_16_xx
1399
	eor	r3,r5,r6			@ a^b, b^c in next round
1400
	ldr	r1,[sp,#6*4]	@ from future BODY_16_xx
1401
#endif
1402
	eor	r0,r0,r5,ror#20	@ Sigma0(a)
1403
	and	r12,r12,r3			@ (b^c)&=(a^b)
1404
	add	r8,r8,r4			@ d+=h
1405
	eor	r12,r12,r6			@ Maj(a,b,c)
1406
	add	r4,r4,r0,ror#2	@ h+=Sigma0(a)
1407
	@ add	r4,r4,r12			@ h+=Maj(a,b,c)
1408
	@ ldr	r2,[sp,#9*4]		@ 24
1409
	@ ldr	r1,[sp,#6*4]
1410
	mov	r0,r2,ror#7
1411
	add	r4,r4,r12			@ h+=Maj(a,b,c) from the past
1412
	mov	r12,r1,ror#17
1413
	eor	r0,r0,r2,ror#18
1414
	eor	r12,r12,r1,ror#19
1415
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1416
	ldr	r2,[sp,#8*4]
1417
	eor	r12,r12,r1,lsr#10	@ sigma1(X[i+14])
1418
	ldr	r1,[sp,#1*4]
1419

1420
	add	r12,r12,r0
1421
	eor	r0,r8,r8,ror#5	@ from BODY_00_15
1422
	add	r2,r2,r12
1423
	eor	r0,r0,r8,ror#19	@ Sigma1(e)
1424
	add	r2,r2,r1			@ X[i]
1425
	ldr	r12,[r14],#4			@ *K256++
1426
	add	r11,r11,r2			@ h+=X[i]
1427
	str	r2,[sp,#8*4]
1428
	eor	r2,r9,r10
1429
	add	r11,r11,r0,ror#6	@ h+=Sigma1(e)
1430
	and	r2,r2,r8
1431
	add	r11,r11,r12			@ h+=K256[i]
1432
	eor	r2,r2,r10			@ Ch(e,f,g)
1433
	eor	r0,r4,r4,ror#11
1434
	add	r11,r11,r2			@ h+=Ch(e,f,g)
1435
#if 24==31
1436
	and	r12,r12,#0xff
1437
	cmp	r12,#0xf2			@ done?
1438
#endif
1439
#if 24<15
1440
# if __ARM_ARCH__>=7
1441
	ldr	r2,[r1],#4			@ prefetch
1442
# else
1443
	ldrb	r2,[r1,#3]
1444
# endif
1445
	eor	r12,r4,r5			@ a^b, b^c in next round
1446
#else
1447
	ldr	r2,[sp,#10*4]		@ from future BODY_16_xx
1448
	eor	r12,r4,r5			@ a^b, b^c in next round
1449
	ldr	r1,[sp,#7*4]	@ from future BODY_16_xx
1450
#endif
1451
	eor	r0,r0,r4,ror#20	@ Sigma0(a)
1452
	and	r3,r3,r12			@ (b^c)&=(a^b)
1453
	add	r7,r7,r11			@ d+=h
1454
	eor	r3,r3,r5			@ Maj(a,b,c)
1455
	add	r11,r11,r0,ror#2	@ h+=Sigma0(a)
1456
	@ add	r11,r11,r3			@ h+=Maj(a,b,c)
1457
	@ ldr	r2,[sp,#10*4]		@ 25
1458
	@ ldr	r1,[sp,#7*4]
1459
	mov	r0,r2,ror#7
1460
	add	r11,r11,r3			@ h+=Maj(a,b,c) from the past
1461
	mov	r3,r1,ror#17
1462
	eor	r0,r0,r2,ror#18
1463
	eor	r3,r3,r1,ror#19
1464
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1465
	ldr	r2,[sp,#9*4]
1466
	eor	r3,r3,r1,lsr#10	@ sigma1(X[i+14])
1467
	ldr	r1,[sp,#2*4]
1468

1469
	add	r3,r3,r0
1470
	eor	r0,r7,r7,ror#5	@ from BODY_00_15
1471
	add	r2,r2,r3
1472
	eor	r0,r0,r7,ror#19	@ Sigma1(e)
1473
	add	r2,r2,r1			@ X[i]
1474
	ldr	r3,[r14],#4			@ *K256++
1475
	add	r10,r10,r2			@ h+=X[i]
1476
	str	r2,[sp,#9*4]
1477
	eor	r2,r8,r9
1478
	add	r10,r10,r0,ror#6	@ h+=Sigma1(e)
1479
	and	r2,r2,r7
1480
	add	r10,r10,r3			@ h+=K256[i]
1481
	eor	r2,r2,r9			@ Ch(e,f,g)
1482
	eor	r0,r11,r11,ror#11
1483
	add	r10,r10,r2			@ h+=Ch(e,f,g)
1484
#if 25==31
1485
	and	r3,r3,#0xff
1486
	cmp	r3,#0xf2			@ done?
1487
#endif
1488
#if 25<15
1489
# if __ARM_ARCH__>=7
1490
	ldr	r2,[r1],#4			@ prefetch
1491
# else
1492
	ldrb	r2,[r1,#3]
1493
# endif
1494
	eor	r3,r11,r4			@ a^b, b^c in next round
1495
#else
1496
	ldr	r2,[sp,#11*4]		@ from future BODY_16_xx
1497
	eor	r3,r11,r4			@ a^b, b^c in next round
1498
	ldr	r1,[sp,#8*4]	@ from future BODY_16_xx
1499
#endif
1500
	eor	r0,r0,r11,ror#20	@ Sigma0(a)
1501
	and	r12,r12,r3			@ (b^c)&=(a^b)
1502
	add	r6,r6,r10			@ d+=h
1503
	eor	r12,r12,r4			@ Maj(a,b,c)
1504
	add	r10,r10,r0,ror#2	@ h+=Sigma0(a)
1505
	@ add	r10,r10,r12			@ h+=Maj(a,b,c)
1506
	@ ldr	r2,[sp,#11*4]		@ 26
1507
	@ ldr	r1,[sp,#8*4]
1508
	mov	r0,r2,ror#7
1509
	add	r10,r10,r12			@ h+=Maj(a,b,c) from the past
1510
	mov	r12,r1,ror#17
1511
	eor	r0,r0,r2,ror#18
1512
	eor	r12,r12,r1,ror#19
1513
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1514
	ldr	r2,[sp,#10*4]
1515
	eor	r12,r12,r1,lsr#10	@ sigma1(X[i+14])
1516
	ldr	r1,[sp,#3*4]
1517

1518
	add	r12,r12,r0
1519
	eor	r0,r6,r6,ror#5	@ from BODY_00_15
1520
	add	r2,r2,r12
1521
	eor	r0,r0,r6,ror#19	@ Sigma1(e)
1522
	add	r2,r2,r1			@ X[i]
1523
	ldr	r12,[r14],#4			@ *K256++
1524
	add	r9,r9,r2			@ h+=X[i]
1525
	str	r2,[sp,#10*4]
1526
	eor	r2,r7,r8
1527
	add	r9,r9,r0,ror#6	@ h+=Sigma1(e)
1528
	and	r2,r2,r6
1529
	add	r9,r9,r12			@ h+=K256[i]
1530
	eor	r2,r2,r8			@ Ch(e,f,g)
1531
	eor	r0,r10,r10,ror#11
1532
	add	r9,r9,r2			@ h+=Ch(e,f,g)
1533
#if 26==31
1534
	and	r12,r12,#0xff
1535
	cmp	r12,#0xf2			@ done?
1536
#endif
1537
#if 26<15
1538
# if __ARM_ARCH__>=7
1539
	ldr	r2,[r1],#4			@ prefetch
1540
# else
1541
	ldrb	r2,[r1,#3]
1542
# endif
1543
	eor	r12,r10,r11			@ a^b, b^c in next round
1544
#else
1545
	ldr	r2,[sp,#12*4]		@ from future BODY_16_xx
1546
	eor	r12,r10,r11			@ a^b, b^c in next round
1547
	ldr	r1,[sp,#9*4]	@ from future BODY_16_xx
1548
#endif
1549
	eor	r0,r0,r10,ror#20	@ Sigma0(a)
1550
	and	r3,r3,r12			@ (b^c)&=(a^b)
1551
	add	r5,r5,r9			@ d+=h
1552
	eor	r3,r3,r11			@ Maj(a,b,c)
1553
	add	r9,r9,r0,ror#2	@ h+=Sigma0(a)
1554
	@ add	r9,r9,r3			@ h+=Maj(a,b,c)
1555
	@ ldr	r2,[sp,#12*4]		@ 27
1556
	@ ldr	r1,[sp,#9*4]
1557
	mov	r0,r2,ror#7
1558
	add	r9,r9,r3			@ h+=Maj(a,b,c) from the past
1559
	mov	r3,r1,ror#17
1560
	eor	r0,r0,r2,ror#18
1561
	eor	r3,r3,r1,ror#19
1562
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1563
	ldr	r2,[sp,#11*4]
1564
	eor	r3,r3,r1,lsr#10	@ sigma1(X[i+14])
1565
	ldr	r1,[sp,#4*4]
1566

1567
	add	r3,r3,r0
1568
	eor	r0,r5,r5,ror#5	@ from BODY_00_15
1569
	add	r2,r2,r3
1570
	eor	r0,r0,r5,ror#19	@ Sigma1(e)
1571
	add	r2,r2,r1			@ X[i]
1572
	ldr	r3,[r14],#4			@ *K256++
1573
	add	r8,r8,r2			@ h+=X[i]
1574
	str	r2,[sp,#11*4]
1575
	eor	r2,r6,r7
1576
	add	r8,r8,r0,ror#6	@ h+=Sigma1(e)
1577
	and	r2,r2,r5
1578
	add	r8,r8,r3			@ h+=K256[i]
1579
	eor	r2,r2,r7			@ Ch(e,f,g)
1580
	eor	r0,r9,r9,ror#11
1581
	add	r8,r8,r2			@ h+=Ch(e,f,g)
1582
#if 27==31
1583
	and	r3,r3,#0xff
1584
	cmp	r3,#0xf2			@ done?
1585
#endif
1586
#if 27<15
1587
# if __ARM_ARCH__>=7
1588
	ldr	r2,[r1],#4			@ prefetch
1589
# else
1590
	ldrb	r2,[r1,#3]
1591
# endif
1592
	eor	r3,r9,r10			@ a^b, b^c in next round
1593
#else
1594
	ldr	r2,[sp,#13*4]		@ from future BODY_16_xx
1595
	eor	r3,r9,r10			@ a^b, b^c in next round
1596
	ldr	r1,[sp,#10*4]	@ from future BODY_16_xx
1597
#endif
1598
	eor	r0,r0,r9,ror#20	@ Sigma0(a)
1599
	and	r12,r12,r3			@ (b^c)&=(a^b)
1600
	add	r4,r4,r8			@ d+=h
1601
	eor	r12,r12,r10			@ Maj(a,b,c)
1602
	add	r8,r8,r0,ror#2	@ h+=Sigma0(a)
1603
	@ add	r8,r8,r12			@ h+=Maj(a,b,c)
1604
	@ ldr	r2,[sp,#13*4]		@ 28
1605
	@ ldr	r1,[sp,#10*4]
1606
	mov	r0,r2,ror#7
1607
	add	r8,r8,r12			@ h+=Maj(a,b,c) from the past
1608
	mov	r12,r1,ror#17
1609
	eor	r0,r0,r2,ror#18
1610
	eor	r12,r12,r1,ror#19
1611
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1612
	ldr	r2,[sp,#12*4]
1613
	eor	r12,r12,r1,lsr#10	@ sigma1(X[i+14])
1614
	ldr	r1,[sp,#5*4]
1615

1616
	add	r12,r12,r0
1617
	eor	r0,r4,r4,ror#5	@ from BODY_00_15
1618
	add	r2,r2,r12
1619
	eor	r0,r0,r4,ror#19	@ Sigma1(e)
1620
	add	r2,r2,r1			@ X[i]
1621
	ldr	r12,[r14],#4			@ *K256++
1622
	add	r7,r7,r2			@ h+=X[i]
1623
	str	r2,[sp,#12*4]
1624
	eor	r2,r5,r6
1625
	add	r7,r7,r0,ror#6	@ h+=Sigma1(e)
1626
	and	r2,r2,r4
1627
	add	r7,r7,r12			@ h+=K256[i]
1628
	eor	r2,r2,r6			@ Ch(e,f,g)
1629
	eor	r0,r8,r8,ror#11
1630
	add	r7,r7,r2			@ h+=Ch(e,f,g)
1631
#if 28==31
1632
	and	r12,r12,#0xff
1633
	cmp	r12,#0xf2			@ done?
1634
#endif
1635
#if 28<15
1636
# if __ARM_ARCH__>=7
1637
	ldr	r2,[r1],#4			@ prefetch
1638
# else
1639
	ldrb	r2,[r1,#3]
1640
# endif
1641
	eor	r12,r8,r9			@ a^b, b^c in next round
1642
#else
1643
	ldr	r2,[sp,#14*4]		@ from future BODY_16_xx
1644
	eor	r12,r8,r9			@ a^b, b^c in next round
1645
	ldr	r1,[sp,#11*4]	@ from future BODY_16_xx
1646
#endif
1647
	eor	r0,r0,r8,ror#20	@ Sigma0(a)
1648
	and	r3,r3,r12			@ (b^c)&=(a^b)
1649
	add	r11,r11,r7			@ d+=h
1650
	eor	r3,r3,r9			@ Maj(a,b,c)
1651
	add	r7,r7,r0,ror#2	@ h+=Sigma0(a)
1652
	@ add	r7,r7,r3			@ h+=Maj(a,b,c)
1653
	@ ldr	r2,[sp,#14*4]		@ 29
1654
	@ ldr	r1,[sp,#11*4]
1655
	mov	r0,r2,ror#7
1656
	add	r7,r7,r3			@ h+=Maj(a,b,c) from the past
1657
	mov	r3,r1,ror#17
1658
	eor	r0,r0,r2,ror#18
1659
	eor	r3,r3,r1,ror#19
1660
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1661
	ldr	r2,[sp,#13*4]
1662
	eor	r3,r3,r1,lsr#10	@ sigma1(X[i+14])
1663
	ldr	r1,[sp,#6*4]
1664

1665
	add	r3,r3,r0
1666
	eor	r0,r11,r11,ror#5	@ from BODY_00_15
1667
	add	r2,r2,r3
1668
	eor	r0,r0,r11,ror#19	@ Sigma1(e)
1669
	add	r2,r2,r1			@ X[i]
1670
	ldr	r3,[r14],#4			@ *K256++
1671
	add	r6,r6,r2			@ h+=X[i]
1672
	str	r2,[sp,#13*4]
1673
	eor	r2,r4,r5
1674
	add	r6,r6,r0,ror#6	@ h+=Sigma1(e)
1675
	and	r2,r2,r11
1676
	add	r6,r6,r3			@ h+=K256[i]
1677
	eor	r2,r2,r5			@ Ch(e,f,g)
1678
	eor	r0,r7,r7,ror#11
1679
	add	r6,r6,r2			@ h+=Ch(e,f,g)
1680
#if 29==31
1681
	and	r3,r3,#0xff
1682
	cmp	r3,#0xf2			@ done?
1683
#endif
1684
#if 29<15
1685
# if __ARM_ARCH__>=7
1686
	ldr	r2,[r1],#4			@ prefetch
1687
# else
1688
	ldrb	r2,[r1,#3]
1689
# endif
1690
	eor	r3,r7,r8			@ a^b, b^c in next round
1691
#else
1692
	ldr	r2,[sp,#15*4]		@ from future BODY_16_xx
1693
	eor	r3,r7,r8			@ a^b, b^c in next round
1694
	ldr	r1,[sp,#12*4]	@ from future BODY_16_xx
1695
#endif
1696
	eor	r0,r0,r7,ror#20	@ Sigma0(a)
1697
	and	r12,r12,r3			@ (b^c)&=(a^b)
1698
	add	r10,r10,r6			@ d+=h
1699
	eor	r12,r12,r8			@ Maj(a,b,c)
1700
	add	r6,r6,r0,ror#2	@ h+=Sigma0(a)
1701
	@ add	r6,r6,r12			@ h+=Maj(a,b,c)
1702
	@ ldr	r2,[sp,#15*4]		@ 30
1703
	@ ldr	r1,[sp,#12*4]
1704
	mov	r0,r2,ror#7
1705
	add	r6,r6,r12			@ h+=Maj(a,b,c) from the past
1706
	mov	r12,r1,ror#17
1707
	eor	r0,r0,r2,ror#18
1708
	eor	r12,r12,r1,ror#19
1709
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1710
	ldr	r2,[sp,#14*4]
1711
	eor	r12,r12,r1,lsr#10	@ sigma1(X[i+14])
1712
	ldr	r1,[sp,#7*4]
1713

1714
	add	r12,r12,r0
1715
	eor	r0,r10,r10,ror#5	@ from BODY_00_15
1716
	add	r2,r2,r12
1717
	eor	r0,r0,r10,ror#19	@ Sigma1(e)
1718
	add	r2,r2,r1			@ X[i]
1719
	ldr	r12,[r14],#4			@ *K256++
1720
	add	r5,r5,r2			@ h+=X[i]
1721
	str	r2,[sp,#14*4]
1722
	eor	r2,r11,r4
1723
	add	r5,r5,r0,ror#6	@ h+=Sigma1(e)
1724
	and	r2,r2,r10
1725
	add	r5,r5,r12			@ h+=K256[i]
1726
	eor	r2,r2,r4			@ Ch(e,f,g)
1727
	eor	r0,r6,r6,ror#11
1728
	add	r5,r5,r2			@ h+=Ch(e,f,g)
1729
#if 30==31
1730
	and	r12,r12,#0xff
1731
	cmp	r12,#0xf2			@ done?
1732
#endif
1733
#if 30<15
1734
# if __ARM_ARCH__>=7
1735
	ldr	r2,[r1],#4			@ prefetch
1736
# else
1737
	ldrb	r2,[r1,#3]
1738
# endif
1739
	eor	r12,r6,r7			@ a^b, b^c in next round
1740
#else
1741
	ldr	r2,[sp,#0*4]		@ from future BODY_16_xx
1742
	eor	r12,r6,r7			@ a^b, b^c in next round
1743
	ldr	r1,[sp,#13*4]	@ from future BODY_16_xx
1744
#endif
1745
	eor	r0,r0,r6,ror#20	@ Sigma0(a)
1746
	and	r3,r3,r12			@ (b^c)&=(a^b)
1747
	add	r9,r9,r5			@ d+=h
1748
	eor	r3,r3,r7			@ Maj(a,b,c)
1749
	add	r5,r5,r0,ror#2	@ h+=Sigma0(a)
1750
	@ add	r5,r5,r3			@ h+=Maj(a,b,c)
1751
	@ ldr	r2,[sp,#0*4]		@ 31
1752
	@ ldr	r1,[sp,#13*4]
1753
	mov	r0,r2,ror#7
1754
	add	r5,r5,r3			@ h+=Maj(a,b,c) from the past
1755
	mov	r3,r1,ror#17
1756
	eor	r0,r0,r2,ror#18
1757
	eor	r3,r3,r1,ror#19
1758
	eor	r0,r0,r2,lsr#3	@ sigma0(X[i+1])
1759
	ldr	r2,[sp,#15*4]
1760
	eor	r3,r3,r1,lsr#10	@ sigma1(X[i+14])
1761
	ldr	r1,[sp,#8*4]
1762

1763
	add	r3,r3,r0
1764
	eor	r0,r9,r9,ror#5	@ from BODY_00_15
1765
	add	r2,r2,r3
1766
	eor	r0,r0,r9,ror#19	@ Sigma1(e)
1767
	add	r2,r2,r1			@ X[i]
1768
	ldr	r3,[r14],#4			@ *K256++
1769
	add	r4,r4,r2			@ h+=X[i]
1770
	str	r2,[sp,#15*4]
1771
	eor	r2,r10,r11
1772
	add	r4,r4,r0,ror#6	@ h+=Sigma1(e)
1773
	and	r2,r2,r9
1774
	add	r4,r4,r3			@ h+=K256[i]
1775
	eor	r2,r2,r11			@ Ch(e,f,g)
1776
	eor	r0,r5,r5,ror#11
1777
	add	r4,r4,r2			@ h+=Ch(e,f,g)
1778
#if 31==31
1779
	and	r3,r3,#0xff
1780
	cmp	r3,#0xf2			@ done?
1781
#endif
1782
#if 31<15
1783
# if __ARM_ARCH__>=7
1784
	ldr	r2,[r1],#4			@ prefetch
1785
# else
1786
	ldrb	r2,[r1,#3]
1787
# endif
1788
	eor	r3,r5,r6			@ a^b, b^c in next round
1789
#else
1790
	ldr	r2,[sp,#1*4]		@ from future BODY_16_xx
1791
	eor	r3,r5,r6			@ a^b, b^c in next round
1792
	ldr	r1,[sp,#14*4]	@ from future BODY_16_xx
1793
#endif
1794
	eor	r0,r0,r5,ror#20	@ Sigma0(a)
1795
	and	r12,r12,r3			@ (b^c)&=(a^b)
1796
	add	r8,r8,r4			@ d+=h
1797
	eor	r12,r12,r6			@ Maj(a,b,c)
1798
	add	r4,r4,r0,ror#2	@ h+=Sigma0(a)
1799
	@ add	r4,r4,r12			@ h+=Maj(a,b,c)
1800
#ifdef	__thumb2__
1801
	ite	eq			@ Thumb2 thing, sanity check in ARM
1802
#endif
1803
	ldreq	r3,[sp,#16*4]		@ pull ctx
1804
	bne	.Lrounds_16_xx
1805

1806
	add	r4,r4,r12		@ h+=Maj(a,b,c) from the past
1807
	ldr	r0,[r3,#0]
1808
	ldr	r2,[r3,#4]
1809
	ldr	r12,[r3,#8]
1810
	add	r4,r4,r0
1811
	ldr	r0,[r3,#12]
1812
	add	r5,r5,r2
1813
	ldr	r2,[r3,#16]
1814
	add	r6,r6,r12
1815
	ldr	r12,[r3,#20]
1816
	add	r7,r7,r0
1817
	ldr	r0,[r3,#24]
1818
	add	r8,r8,r2
1819
	ldr	r2,[r3,#28]
1820
	add	r9,r9,r12
1821
	ldr	r1,[sp,#17*4]		@ pull inp
1822
	ldr	r12,[sp,#18*4]		@ pull inp+len
1823
	add	r10,r10,r0
1824
	add	r11,r11,r2
1825
	stmia	r3,{r4,r5,r6,r7,r8,r9,r10,r11}
1826
	cmp	r1,r12
1827
	sub	r14,r14,#256	@ rewind Ktbl
1828
	bne	.Loop
1829

1830
	add	sp,sp,#19*4	@ destroy frame
1831
#if __ARM_ARCH__>=5
1832
	ldmia	sp!,{r4-r11,pc}
1833
#else
1834
	ldmia	sp!,{r4-r11,lr}
1835
	tst	lr,#1
1836
	moveq	pc,lr			@ be binary compatible with V4, yet
1837
	.word	0xe12fff1e			@ interoperable with Thumb ISA:-)
1838
#endif
1839
.size	zfs_sha256_block_armv7,.-zfs_sha256_block_armv7
1840

1841
#if __ARM_ARCH__ >= 7
1842
.arch	armv7-a
1843
.fpu	neon
1844

1845
.globl	zfs_sha256_block_neon
1846
.type	zfs_sha256_block_neon,%function
1847
.align	5
1848
.skip	16
1849
zfs_sha256_block_neon:
1850
.LNEON:
1851
	stmdb	sp!,{r4-r12,lr}
1852

1853
	sub	r11,sp,#16*4+16
1854
	adr	r14,K256
1855
	bic	r11,r11,#15		@ align for 128-bit stores
1856
	mov	r12,sp
1857
	mov	sp,r11			@ alloca
1858
	add	r2,r1,r2,lsl#6	@ len to point at the end of inp
1859

1860
	vld1.8		{q0},[r1]!
1861
	vld1.8		{q1},[r1]!
1862
	vld1.8		{q2},[r1]!
1863
	vld1.8		{q3},[r1]!
1864
	vld1.32		{q8},[r14,:128]!
1865
	vld1.32		{q9},[r14,:128]!
1866
	vld1.32		{q10},[r14,:128]!
1867
	vld1.32		{q11},[r14,:128]!
1868
	vrev32.8	q0,q0		@ yes, even on
1869
	str		r0,[sp,#64]
1870
	vrev32.8	q1,q1		@ big-endian
1871
	str		r1,[sp,#68]
1872
	mov		r1,sp
1873
	vrev32.8	q2,q2
1874
	str		r2,[sp,#72]
1875
	vrev32.8	q3,q3
1876
	str		r12,[sp,#76]		@ save original sp
1877
	vadd.i32	q8,q8,q0
1878
	vadd.i32	q9,q9,q1
1879
	vst1.32		{q8},[r1,:128]!
1880
	vadd.i32	q10,q10,q2
1881
	vst1.32		{q9},[r1,:128]!
1882
	vadd.i32	q11,q11,q3
1883
	vst1.32		{q10},[r1,:128]!
1884
	vst1.32		{q11},[r1,:128]!
1885

1886
	ldmia		r0,{r4-r11}
1887
	sub		r1,r1,#64
1888
	ldr		r2,[sp,#0]
1889
	eor		r12,r12,r12
1890
	eor		r3,r5,r6
1891
	b		.L_00_48
1892

1893
.align	4
1894
.L_00_48:
1895
	vext.8	q8,q0,q1,#4
1896
	add	r11,r11,r2
1897
	eor	r2,r9,r10
1898
	eor	r0,r8,r8,ror#5
1899
	vext.8	q9,q2,q3,#4
1900
	add	r4,r4,r12
1901
	and	r2,r2,r8
1902
	eor	r12,r0,r8,ror#19
1903
	vshr.u32	q10,q8,#7
1904
	eor	r0,r4,r4,ror#11
1905
	eor	r2,r2,r10
1906
	vadd.i32	q0,q0,q9
1907
	add	r11,r11,r12,ror#6
1908
	eor	r12,r4,r5
1909
	vshr.u32	q9,q8,#3
1910
	eor	r0,r0,r4,ror#20
1911
	add	r11,r11,r2
1912
	vsli.32	q10,q8,#25
1913
	ldr	r2,[sp,#4]
1914
	and	r3,r3,r12
1915
	vshr.u32	q11,q8,#18
1916
	add	r7,r7,r11
1917
	add	r11,r11,r0,ror#2
1918
	eor	r3,r3,r5
1919
	veor	q9,q9,q10
1920
	add	r10,r10,r2
1921
	vsli.32	q11,q8,#14
1922
	eor	r2,r8,r9
1923
	eor	r0,r7,r7,ror#5
1924
	vshr.u32	d24,d7,#17
1925
	add	r11,r11,r3
1926
	and	r2,r2,r7
1927
	veor	q9,q9,q11
1928
	eor	r3,r0,r7,ror#19
1929
	eor	r0,r11,r11,ror#11
1930
	vsli.32	d24,d7,#15
1931
	eor	r2,r2,r9
1932
	add	r10,r10,r3,ror#6
1933
	vshr.u32	d25,d7,#10
1934
	eor	r3,r11,r4
1935
	eor	r0,r0,r11,ror#20
1936
	vadd.i32	q0,q0,q9
1937
	add	r10,r10,r2
1938
	ldr	r2,[sp,#8]
1939
	veor	d25,d25,d24
1940
	and	r12,r12,r3
1941
	add	r6,r6,r10
1942
	vshr.u32	d24,d7,#19
1943
	add	r10,r10,r0,ror#2
1944
	eor	r12,r12,r4
1945
	vsli.32	d24,d7,#13
1946
	add	r9,r9,r2
1947
	eor	r2,r7,r8
1948
	veor	d25,d25,d24
1949
	eor	r0,r6,r6,ror#5
1950
	add	r10,r10,r12
1951
	vadd.i32	d0,d0,d25
1952
	and	r2,r2,r6
1953
	eor	r12,r0,r6,ror#19
1954
	vshr.u32	d24,d0,#17
1955
	eor	r0,r10,r10,ror#11
1956
	eor	r2,r2,r8
1957
	vsli.32	d24,d0,#15
1958
	add	r9,r9,r12,ror#6
1959
	eor	r12,r10,r11
1960
	vshr.u32	d25,d0,#10
1961
	eor	r0,r0,r10,ror#20
1962
	add	r9,r9,r2
1963
	veor	d25,d25,d24
1964
	ldr	r2,[sp,#12]
1965
	and	r3,r3,r12
1966
	vshr.u32	d24,d0,#19
1967
	add	r5,r5,r9
1968
	add	r9,r9,r0,ror#2
1969
	eor	r3,r3,r11
1970
	vld1.32	{q8},[r14,:128]!
1971
	add	r8,r8,r2
1972
	vsli.32	d24,d0,#13
1973
	eor	r2,r6,r7
1974
	eor	r0,r5,r5,ror#5
1975
	veor	d25,d25,d24
1976
	add	r9,r9,r3
1977
	and	r2,r2,r5
1978
	vadd.i32	d1,d1,d25
1979
	eor	r3,r0,r5,ror#19
1980
	eor	r0,r9,r9,ror#11
1981
	vadd.i32	q8,q8,q0
1982
	eor	r2,r2,r7
1983
	add	r8,r8,r3,ror#6
1984
	eor	r3,r9,r10
1985
	eor	r0,r0,r9,ror#20
1986
	add	r8,r8,r2
1987
	ldr	r2,[sp,#16]
1988
	and	r12,r12,r3
1989
	add	r4,r4,r8
1990
	vst1.32	{q8},[r1,:128]!
1991
	add	r8,r8,r0,ror#2
1992
	eor	r12,r12,r10
1993
	vext.8	q8,q1,q2,#4
1994
	add	r7,r7,r2
1995
	eor	r2,r5,r6
1996
	eor	r0,r4,r4,ror#5
1997
	vext.8	q9,q3,q0,#4
1998
	add	r8,r8,r12
1999
	and	r2,r2,r4
2000
	eor	r12,r0,r4,ror#19
2001
	vshr.u32	q10,q8,#7
2002
	eor	r0,r8,r8,ror#11
2003
	eor	r2,r2,r6
2004
	vadd.i32	q1,q1,q9
2005
	add	r7,r7,r12,ror#6
2006
	eor	r12,r8,r9
2007
	vshr.u32	q9,q8,#3
2008
	eor	r0,r0,r8,ror#20
2009
	add	r7,r7,r2
2010
	vsli.32	q10,q8,#25
2011
	ldr	r2,[sp,#20]
2012
	and	r3,r3,r12
2013
	vshr.u32	q11,q8,#18
2014
	add	r11,r11,r7
2015
	add	r7,r7,r0,ror#2
2016
	eor	r3,r3,r9
2017
	veor	q9,q9,q10
2018
	add	r6,r6,r2
2019
	vsli.32	q11,q8,#14
2020
	eor	r2,r4,r5
2021
	eor	r0,r11,r11,ror#5
2022
	vshr.u32	d24,d1,#17
2023
	add	r7,r7,r3
2024
	and	r2,r2,r11
2025
	veor	q9,q9,q11
2026
	eor	r3,r0,r11,ror#19
2027
	eor	r0,r7,r7,ror#11
2028
	vsli.32	d24,d1,#15
2029
	eor	r2,r2,r5
2030
	add	r6,r6,r3,ror#6
2031
	vshr.u32	d25,d1,#10
2032
	eor	r3,r7,r8
2033
	eor	r0,r0,r7,ror#20
2034
	vadd.i32	q1,q1,q9
2035
	add	r6,r6,r2
2036
	ldr	r2,[sp,#24]
2037
	veor	d25,d25,d24
2038
	and	r12,r12,r3
2039
	add	r10,r10,r6
2040
	vshr.u32	d24,d1,#19
2041
	add	r6,r6,r0,ror#2
2042
	eor	r12,r12,r8
2043
	vsli.32	d24,d1,#13
2044
	add	r5,r5,r2
2045
	eor	r2,r11,r4
2046
	veor	d25,d25,d24
2047
	eor	r0,r10,r10,ror#5
2048
	add	r6,r6,r12
2049
	vadd.i32	d2,d2,d25
2050
	and	r2,r2,r10
2051
	eor	r12,r0,r10,ror#19
2052
	vshr.u32	d24,d2,#17
2053
	eor	r0,r6,r6,ror#11
2054
	eor	r2,r2,r4
2055
	vsli.32	d24,d2,#15
2056
	add	r5,r5,r12,ror#6
2057
	eor	r12,r6,r7
2058
	vshr.u32	d25,d2,#10
2059
	eor	r0,r0,r6,ror#20
2060
	add	r5,r5,r2
2061
	veor	d25,d25,d24
2062
	ldr	r2,[sp,#28]
2063
	and	r3,r3,r12
2064
	vshr.u32	d24,d2,#19
2065
	add	r9,r9,r5
2066
	add	r5,r5,r0,ror#2
2067
	eor	r3,r3,r7
2068
	vld1.32	{q8},[r14,:128]!
2069
	add	r4,r4,r2
2070
	vsli.32	d24,d2,#13
2071
	eor	r2,r10,r11
2072
	eor	r0,r9,r9,ror#5
2073
	veor	d25,d25,d24
2074
	add	r5,r5,r3
2075
	and	r2,r2,r9
2076
	vadd.i32	d3,d3,d25
2077
	eor	r3,r0,r9,ror#19
2078
	eor	r0,r5,r5,ror#11
2079
	vadd.i32	q8,q8,q1
2080
	eor	r2,r2,r11
2081
	add	r4,r4,r3,ror#6
2082
	eor	r3,r5,r6
2083
	eor	r0,r0,r5,ror#20
2084
	add	r4,r4,r2
2085
	ldr	r2,[sp,#32]
2086
	and	r12,r12,r3
2087
	add	r8,r8,r4
2088
	vst1.32	{q8},[r1,:128]!
2089
	add	r4,r4,r0,ror#2
2090
	eor	r12,r12,r6
2091
	vext.8	q8,q2,q3,#4
2092
	add	r11,r11,r2
2093
	eor	r2,r9,r10
2094
	eor	r0,r8,r8,ror#5
2095
	vext.8	q9,q0,q1,#4
2096
	add	r4,r4,r12
2097
	and	r2,r2,r8
2098
	eor	r12,r0,r8,ror#19
2099
	vshr.u32	q10,q8,#7
2100
	eor	r0,r4,r4,ror#11
2101
	eor	r2,r2,r10
2102
	vadd.i32	q2,q2,q9
2103
	add	r11,r11,r12,ror#6
2104
	eor	r12,r4,r5
2105
	vshr.u32	q9,q8,#3
2106
	eor	r0,r0,r4,ror#20
2107
	add	r11,r11,r2
2108
	vsli.32	q10,q8,#25
2109
	ldr	r2,[sp,#36]
2110
	and	r3,r3,r12
2111
	vshr.u32	q11,q8,#18
2112
	add	r7,r7,r11
2113
	add	r11,r11,r0,ror#2
2114
	eor	r3,r3,r5
2115
	veor	q9,q9,q10
2116
	add	r10,r10,r2
2117
	vsli.32	q11,q8,#14
2118
	eor	r2,r8,r9
2119
	eor	r0,r7,r7,ror#5
2120
	vshr.u32	d24,d3,#17
2121
	add	r11,r11,r3
2122
	and	r2,r2,r7
2123
	veor	q9,q9,q11
2124
	eor	r3,r0,r7,ror#19
2125
	eor	r0,r11,r11,ror#11
2126
	vsli.32	d24,d3,#15
2127
	eor	r2,r2,r9
2128
	add	r10,r10,r3,ror#6
2129
	vshr.u32	d25,d3,#10
2130
	eor	r3,r11,r4
2131
	eor	r0,r0,r11,ror#20
2132
	vadd.i32	q2,q2,q9
2133
	add	r10,r10,r2
2134
	ldr	r2,[sp,#40]
2135
	veor	d25,d25,d24
2136
	and	r12,r12,r3
2137
	add	r6,r6,r10
2138
	vshr.u32	d24,d3,#19
2139
	add	r10,r10,r0,ror#2
2140
	eor	r12,r12,r4
2141
	vsli.32	d24,d3,#13
2142
	add	r9,r9,r2
2143
	eor	r2,r7,r8
2144
	veor	d25,d25,d24
2145
	eor	r0,r6,r6,ror#5
2146
	add	r10,r10,r12
2147
	vadd.i32	d4,d4,d25
2148
	and	r2,r2,r6
2149
	eor	r12,r0,r6,ror#19
2150
	vshr.u32	d24,d4,#17
2151
	eor	r0,r10,r10,ror#11
2152
	eor	r2,r2,r8
2153
	vsli.32	d24,d4,#15
2154
	add	r9,r9,r12,ror#6
2155
	eor	r12,r10,r11
2156
	vshr.u32	d25,d4,#10
2157
	eor	r0,r0,r10,ror#20
2158
	add	r9,r9,r2
2159
	veor	d25,d25,d24
2160
	ldr	r2,[sp,#44]
2161
	and	r3,r3,r12
2162
	vshr.u32	d24,d4,#19
2163
	add	r5,r5,r9
2164
	add	r9,r9,r0,ror#2
2165
	eor	r3,r3,r11
2166
	vld1.32	{q8},[r14,:128]!
2167
	add	r8,r8,r2
2168
	vsli.32	d24,d4,#13
2169
	eor	r2,r6,r7
2170
	eor	r0,r5,r5,ror#5
2171
	veor	d25,d25,d24
2172
	add	r9,r9,r3
2173
	and	r2,r2,r5
2174
	vadd.i32	d5,d5,d25
2175
	eor	r3,r0,r5,ror#19
2176
	eor	r0,r9,r9,ror#11
2177
	vadd.i32	q8,q8,q2
2178
	eor	r2,r2,r7
2179
	add	r8,r8,r3,ror#6
2180
	eor	r3,r9,r10
2181
	eor	r0,r0,r9,ror#20
2182
	add	r8,r8,r2
2183
	ldr	r2,[sp,#48]
2184
	and	r12,r12,r3
2185
	add	r4,r4,r8
2186
	vst1.32	{q8},[r1,:128]!
2187
	add	r8,r8,r0,ror#2
2188
	eor	r12,r12,r10
2189
	vext.8	q8,q3,q0,#4
2190
	add	r7,r7,r2
2191
	eor	r2,r5,r6
2192
	eor	r0,r4,r4,ror#5
2193
	vext.8	q9,q1,q2,#4
2194
	add	r8,r8,r12
2195
	and	r2,r2,r4
2196
	eor	r12,r0,r4,ror#19
2197
	vshr.u32	q10,q8,#7
2198
	eor	r0,r8,r8,ror#11
2199
	eor	r2,r2,r6
2200
	vadd.i32	q3,q3,q9
2201
	add	r7,r7,r12,ror#6
2202
	eor	r12,r8,r9
2203
	vshr.u32	q9,q8,#3
2204
	eor	r0,r0,r8,ror#20
2205
	add	r7,r7,r2
2206
	vsli.32	q10,q8,#25
2207
	ldr	r2,[sp,#52]
2208
	and	r3,r3,r12
2209
	vshr.u32	q11,q8,#18
2210
	add	r11,r11,r7
2211
	add	r7,r7,r0,ror#2
2212
	eor	r3,r3,r9
2213
	veor	q9,q9,q10
2214
	add	r6,r6,r2
2215
	vsli.32	q11,q8,#14
2216
	eor	r2,r4,r5
2217
	eor	r0,r11,r11,ror#5
2218
	vshr.u32	d24,d5,#17
2219
	add	r7,r7,r3
2220
	and	r2,r2,r11
2221
	veor	q9,q9,q11
2222
	eor	r3,r0,r11,ror#19
2223
	eor	r0,r7,r7,ror#11
2224
	vsli.32	d24,d5,#15
2225
	eor	r2,r2,r5
2226
	add	r6,r6,r3,ror#6
2227
	vshr.u32	d25,d5,#10
2228
	eor	r3,r7,r8
2229
	eor	r0,r0,r7,ror#20
2230
	vadd.i32	q3,q3,q9
2231
	add	r6,r6,r2
2232
	ldr	r2,[sp,#56]
2233
	veor	d25,d25,d24
2234
	and	r12,r12,r3
2235
	add	r10,r10,r6
2236
	vshr.u32	d24,d5,#19
2237
	add	r6,r6,r0,ror#2
2238
	eor	r12,r12,r8
2239
	vsli.32	d24,d5,#13
2240
	add	r5,r5,r2
2241
	eor	r2,r11,r4
2242
	veor	d25,d25,d24
2243
	eor	r0,r10,r10,ror#5
2244
	add	r6,r6,r12
2245
	vadd.i32	d6,d6,d25
2246
	and	r2,r2,r10
2247
	eor	r12,r0,r10,ror#19
2248
	vshr.u32	d24,d6,#17
2249
	eor	r0,r6,r6,ror#11
2250
	eor	r2,r2,r4
2251
	vsli.32	d24,d6,#15
2252
	add	r5,r5,r12,ror#6
2253
	eor	r12,r6,r7
2254
	vshr.u32	d25,d6,#10
2255
	eor	r0,r0,r6,ror#20
2256
	add	r5,r5,r2
2257
	veor	d25,d25,d24
2258
	ldr	r2,[sp,#60]
2259
	and	r3,r3,r12
2260
	vshr.u32	d24,d6,#19
2261
	add	r9,r9,r5
2262
	add	r5,r5,r0,ror#2
2263
	eor	r3,r3,r7
2264
	vld1.32	{q8},[r14,:128]!
2265
	add	r4,r4,r2
2266
	vsli.32	d24,d6,#13
2267
	eor	r2,r10,r11
2268
	eor	r0,r9,r9,ror#5
2269
	veor	d25,d25,d24
2270
	add	r5,r5,r3
2271
	and	r2,r2,r9
2272
	vadd.i32	d7,d7,d25
2273
	eor	r3,r0,r9,ror#19
2274
	eor	r0,r5,r5,ror#11
2275
	vadd.i32	q8,q8,q3
2276
	eor	r2,r2,r11
2277
	add	r4,r4,r3,ror#6
2278
	eor	r3,r5,r6
2279
	eor	r0,r0,r5,ror#20
2280
	add	r4,r4,r2
2281
	ldr	r2,[r14]
2282
	and	r12,r12,r3
2283
	add	r8,r8,r4
2284
	vst1.32	{q8},[r1,:128]!
2285
	add	r4,r4,r0,ror#2
2286
	eor	r12,r12,r6
2287
	teq	r2,#0				@ check for K256 terminator
2288
	ldr	r2,[sp,#0]
2289
	sub	r1,r1,#64
2290
	bne	.L_00_48
2291

2292
	ldr		r1,[sp,#68]
2293
	ldr		r0,[sp,#72]
2294
	sub		r14,r14,#256	@ rewind r14
2295
	teq		r1,r0
2296
	it		eq
2297
	subeq		r1,r1,#64		@ avoid SEGV
2298
	vld1.8		{q0},[r1]!		@ load next input block
2299
	vld1.8		{q1},[r1]!
2300
	vld1.8		{q2},[r1]!
2301
	vld1.8		{q3},[r1]!
2302
	it		ne
2303
	strne		r1,[sp,#68]
2304
	mov		r1,sp
2305
	add	r11,r11,r2
2306
	eor	r2,r9,r10
2307
	eor	r0,r8,r8,ror#5
2308
	add	r4,r4,r12
2309
	vld1.32	{q8},[r14,:128]!
2310
	and	r2,r2,r8
2311
	eor	r12,r0,r8,ror#19
2312
	eor	r0,r4,r4,ror#11
2313
	eor	r2,r2,r10
2314
	vrev32.8	q0,q0
2315
	add	r11,r11,r12,ror#6
2316
	eor	r12,r4,r5
2317
	eor	r0,r0,r4,ror#20
2318
	add	r11,r11,r2
2319
	vadd.i32	q8,q8,q0
2320
	ldr	r2,[sp,#4]
2321
	and	r3,r3,r12
2322
	add	r7,r7,r11
2323
	add	r11,r11,r0,ror#2
2324
	eor	r3,r3,r5
2325
	add	r10,r10,r2
2326
	eor	r2,r8,r9
2327
	eor	r0,r7,r7,ror#5
2328
	add	r11,r11,r3
2329
	and	r2,r2,r7
2330
	eor	r3,r0,r7,ror#19
2331
	eor	r0,r11,r11,ror#11
2332
	eor	r2,r2,r9
2333
	add	r10,r10,r3,ror#6
2334
	eor	r3,r11,r4
2335
	eor	r0,r0,r11,ror#20
2336
	add	r10,r10,r2
2337
	ldr	r2,[sp,#8]
2338
	and	r12,r12,r3
2339
	add	r6,r6,r10
2340
	add	r10,r10,r0,ror#2
2341
	eor	r12,r12,r4
2342
	add	r9,r9,r2
2343
	eor	r2,r7,r8
2344
	eor	r0,r6,r6,ror#5
2345
	add	r10,r10,r12
2346
	and	r2,r2,r6
2347
	eor	r12,r0,r6,ror#19
2348
	eor	r0,r10,r10,ror#11
2349
	eor	r2,r2,r8
2350
	add	r9,r9,r12,ror#6
2351
	eor	r12,r10,r11
2352
	eor	r0,r0,r10,ror#20
2353
	add	r9,r9,r2
2354
	ldr	r2,[sp,#12]
2355
	and	r3,r3,r12
2356
	add	r5,r5,r9
2357
	add	r9,r9,r0,ror#2
2358
	eor	r3,r3,r11
2359
	add	r8,r8,r2
2360
	eor	r2,r6,r7
2361
	eor	r0,r5,r5,ror#5
2362
	add	r9,r9,r3
2363
	and	r2,r2,r5
2364
	eor	r3,r0,r5,ror#19
2365
	eor	r0,r9,r9,ror#11
2366
	eor	r2,r2,r7
2367
	add	r8,r8,r3,ror#6
2368
	eor	r3,r9,r10
2369
	eor	r0,r0,r9,ror#20
2370
	add	r8,r8,r2
2371
	ldr	r2,[sp,#16]
2372
	and	r12,r12,r3
2373
	add	r4,r4,r8
2374
	add	r8,r8,r0,ror#2
2375
	eor	r12,r12,r10
2376
	vst1.32	{q8},[r1,:128]!
2377
	add	r7,r7,r2
2378
	eor	r2,r5,r6
2379
	eor	r0,r4,r4,ror#5
2380
	add	r8,r8,r12
2381
	vld1.32	{q8},[r14,:128]!
2382
	and	r2,r2,r4
2383
	eor	r12,r0,r4,ror#19
2384
	eor	r0,r8,r8,ror#11
2385
	eor	r2,r2,r6
2386
	vrev32.8	q1,q1
2387
	add	r7,r7,r12,ror#6
2388
	eor	r12,r8,r9
2389
	eor	r0,r0,r8,ror#20
2390
	add	r7,r7,r2
2391
	vadd.i32	q8,q8,q1
2392
	ldr	r2,[sp,#20]
2393
	and	r3,r3,r12
2394
	add	r11,r11,r7
2395
	add	r7,r7,r0,ror#2
2396
	eor	r3,r3,r9
2397
	add	r6,r6,r2
2398
	eor	r2,r4,r5
2399
	eor	r0,r11,r11,ror#5
2400
	add	r7,r7,r3
2401
	and	r2,r2,r11
2402
	eor	r3,r0,r11,ror#19
2403
	eor	r0,r7,r7,ror#11
2404
	eor	r2,r2,r5
2405
	add	r6,r6,r3,ror#6
2406
	eor	r3,r7,r8
2407
	eor	r0,r0,r7,ror#20
2408
	add	r6,r6,r2
2409
	ldr	r2,[sp,#24]
2410
	and	r12,r12,r3
2411
	add	r10,r10,r6
2412
	add	r6,r6,r0,ror#2
2413
	eor	r12,r12,r8
2414
	add	r5,r5,r2
2415
	eor	r2,r11,r4
2416
	eor	r0,r10,r10,ror#5
2417
	add	r6,r6,r12
2418
	and	r2,r2,r10
2419
	eor	r12,r0,r10,ror#19
2420
	eor	r0,r6,r6,ror#11
2421
	eor	r2,r2,r4
2422
	add	r5,r5,r12,ror#6
2423
	eor	r12,r6,r7
2424
	eor	r0,r0,r6,ror#20
2425
	add	r5,r5,r2
2426
	ldr	r2,[sp,#28]
2427
	and	r3,r3,r12
2428
	add	r9,r9,r5
2429
	add	r5,r5,r0,ror#2
2430
	eor	r3,r3,r7
2431
	add	r4,r4,r2
2432
	eor	r2,r10,r11
2433
	eor	r0,r9,r9,ror#5
2434
	add	r5,r5,r3
2435
	and	r2,r2,r9
2436
	eor	r3,r0,r9,ror#19
2437
	eor	r0,r5,r5,ror#11
2438
	eor	r2,r2,r11
2439
	add	r4,r4,r3,ror#6
2440
	eor	r3,r5,r6
2441
	eor	r0,r0,r5,ror#20
2442
	add	r4,r4,r2
2443
	ldr	r2,[sp,#32]
2444
	and	r12,r12,r3
2445
	add	r8,r8,r4
2446
	add	r4,r4,r0,ror#2
2447
	eor	r12,r12,r6
2448
	vst1.32	{q8},[r1,:128]!
2449
	add	r11,r11,r2
2450
	eor	r2,r9,r10
2451
	eor	r0,r8,r8,ror#5
2452
	add	r4,r4,r12
2453
	vld1.32	{q8},[r14,:128]!
2454
	and	r2,r2,r8
2455
	eor	r12,r0,r8,ror#19
2456
	eor	r0,r4,r4,ror#11
2457
	eor	r2,r2,r10
2458
	vrev32.8	q2,q2
2459
	add	r11,r11,r12,ror#6
2460
	eor	r12,r4,r5
2461
	eor	r0,r0,r4,ror#20
2462
	add	r11,r11,r2
2463
	vadd.i32	q8,q8,q2
2464
	ldr	r2,[sp,#36]
2465
	and	r3,r3,r12
2466
	add	r7,r7,r11
2467
	add	r11,r11,r0,ror#2
2468
	eor	r3,r3,r5
2469
	add	r10,r10,r2
2470
	eor	r2,r8,r9
2471
	eor	r0,r7,r7,ror#5
2472
	add	r11,r11,r3
2473
	and	r2,r2,r7
2474
	eor	r3,r0,r7,ror#19
2475
	eor	r0,r11,r11,ror#11
2476
	eor	r2,r2,r9
2477
	add	r10,r10,r3,ror#6
2478
	eor	r3,r11,r4
2479
	eor	r0,r0,r11,ror#20
2480
	add	r10,r10,r2
2481
	ldr	r2,[sp,#40]
2482
	and	r12,r12,r3
2483
	add	r6,r6,r10
2484
	add	r10,r10,r0,ror#2
2485
	eor	r12,r12,r4
2486
	add	r9,r9,r2
2487
	eor	r2,r7,r8
2488
	eor	r0,r6,r6,ror#5
2489
	add	r10,r10,r12
2490
	and	r2,r2,r6
2491
	eor	r12,r0,r6,ror#19
2492
	eor	r0,r10,r10,ror#11
2493
	eor	r2,r2,r8
2494
	add	r9,r9,r12,ror#6
2495
	eor	r12,r10,r11
2496
	eor	r0,r0,r10,ror#20
2497
	add	r9,r9,r2
2498
	ldr	r2,[sp,#44]
2499
	and	r3,r3,r12
2500
	add	r5,r5,r9
2501
	add	r9,r9,r0,ror#2
2502
	eor	r3,r3,r11
2503
	add	r8,r8,r2
2504
	eor	r2,r6,r7
2505
	eor	r0,r5,r5,ror#5
2506
	add	r9,r9,r3
2507
	and	r2,r2,r5
2508
	eor	r3,r0,r5,ror#19
2509
	eor	r0,r9,r9,ror#11
2510
	eor	r2,r2,r7
2511
	add	r8,r8,r3,ror#6
2512
	eor	r3,r9,r10
2513
	eor	r0,r0,r9,ror#20
2514
	add	r8,r8,r2
2515
	ldr	r2,[sp,#48]
2516
	and	r12,r12,r3
2517
	add	r4,r4,r8
2518
	add	r8,r8,r0,ror#2
2519
	eor	r12,r12,r10
2520
	vst1.32	{q8},[r1,:128]!
2521
	add	r7,r7,r2
2522
	eor	r2,r5,r6
2523
	eor	r0,r4,r4,ror#5
2524
	add	r8,r8,r12
2525
	vld1.32	{q8},[r14,:128]!
2526
	and	r2,r2,r4
2527
	eor	r12,r0,r4,ror#19
2528
	eor	r0,r8,r8,ror#11
2529
	eor	r2,r2,r6
2530
	vrev32.8	q3,q3
2531
	add	r7,r7,r12,ror#6
2532
	eor	r12,r8,r9
2533
	eor	r0,r0,r8,ror#20
2534
	add	r7,r7,r2
2535
	vadd.i32	q8,q8,q3
2536
	ldr	r2,[sp,#52]
2537
	and	r3,r3,r12
2538
	add	r11,r11,r7
2539
	add	r7,r7,r0,ror#2
2540
	eor	r3,r3,r9
2541
	add	r6,r6,r2
2542
	eor	r2,r4,r5
2543
	eor	r0,r11,r11,ror#5
2544
	add	r7,r7,r3
2545
	and	r2,r2,r11
2546
	eor	r3,r0,r11,ror#19
2547
	eor	r0,r7,r7,ror#11
2548
	eor	r2,r2,r5
2549
	add	r6,r6,r3,ror#6
2550
	eor	r3,r7,r8
2551
	eor	r0,r0,r7,ror#20
2552
	add	r6,r6,r2
2553
	ldr	r2,[sp,#56]
2554
	and	r12,r12,r3
2555
	add	r10,r10,r6
2556
	add	r6,r6,r0,ror#2
2557
	eor	r12,r12,r8
2558
	add	r5,r5,r2
2559
	eor	r2,r11,r4
2560
	eor	r0,r10,r10,ror#5
2561
	add	r6,r6,r12
2562
	and	r2,r2,r10
2563
	eor	r12,r0,r10,ror#19
2564
	eor	r0,r6,r6,ror#11
2565
	eor	r2,r2,r4
2566
	add	r5,r5,r12,ror#6
2567
	eor	r12,r6,r7
2568
	eor	r0,r0,r6,ror#20
2569
	add	r5,r5,r2
2570
	ldr	r2,[sp,#60]
2571
	and	r3,r3,r12
2572
	add	r9,r9,r5
2573
	add	r5,r5,r0,ror#2
2574
	eor	r3,r3,r7
2575
	add	r4,r4,r2
2576
	eor	r2,r10,r11
2577
	eor	r0,r9,r9,ror#5
2578
	add	r5,r5,r3
2579
	and	r2,r2,r9
2580
	eor	r3,r0,r9,ror#19
2581
	eor	r0,r5,r5,ror#11
2582
	eor	r2,r2,r11
2583
	add	r4,r4,r3,ror#6
2584
	eor	r3,r5,r6
2585
	eor	r0,r0,r5,ror#20
2586
	add	r4,r4,r2
2587
	ldr	r2,[sp,#64]
2588
	and	r12,r12,r3
2589
	add	r8,r8,r4
2590
	add	r4,r4,r0,ror#2
2591
	eor	r12,r12,r6
2592
	vst1.32	{q8},[r1,:128]!
2593
	ldr	r0,[r2,#0]
2594
	add	r4,r4,r12			@ h+=Maj(a,b,c) from the past
2595
	ldr	r12,[r2,#4]
2596
	ldr	r3,[r2,#8]
2597
	ldr	r1,[r2,#12]
2598
	add	r4,r4,r0			@ accumulate
2599
	ldr	r0,[r2,#16]
2600
	add	r5,r5,r12
2601
	ldr	r12,[r2,#20]
2602
	add	r6,r6,r3
2603
	ldr	r3,[r2,#24]
2604
	add	r7,r7,r1
2605
	ldr	r1,[r2,#28]
2606
	add	r8,r8,r0
2607
	str	r4,[r2],#4
2608
	add	r9,r9,r12
2609
	str	r5,[r2],#4
2610
	add	r10,r10,r3
2611
	str	r6,[r2],#4
2612
	add	r11,r11,r1
2613
	str	r7,[r2],#4
2614
	stmia	r2,{r8-r11}
2615

2616
	ittte	ne
2617
	movne	r1,sp
2618
	ldrne	r2,[sp,#0]
2619
	eorne	r12,r12,r12
2620
	ldreq	sp,[sp,#76]			@ restore original sp
2621
	itt	ne
2622
	eorne	r3,r5,r6
2623
	bne	.L_00_48
2624

2625
	ldmia	sp!,{r4-r12,pc}
2626
.size	zfs_sha256_block_neon,.-zfs_sha256_block_neon
2627

2628
# if defined(__thumb2__)
2629
#  define INST(a,b,c,d)	.byte	c,d|0xc,a,b
2630
# else
2631
#  define INST(a,b,c,d)	.byte	a,b,c,d
2632
# endif
2633

2634
.globl	zfs_sha256_block_armv8
2635
.type	zfs_sha256_block_armv8,%function
2636
.align	5
2637
zfs_sha256_block_armv8:
2638
.LARMv8:
2639
	vld1.32	{q0,q1},[r0]
2640
	sub	r3,r3,#256+32
2641
	add	r2,r1,r2,lsl#6	@ len to point at the end of inp
2642
	b	.Loop_v8
2643

2644
.align	4
2645
.Loop_v8:
2646
	vld1.8		{q8-q9},[r1]!
2647
	vld1.8		{q10-q11},[r1]!
2648
	vld1.32		{q12},[r3]!
2649
	vrev32.8	q8,q8
2650
	vrev32.8	q9,q9
2651
	vrev32.8	q10,q10
2652
	vrev32.8	q11,q11
2653
	vmov		q14,q0	@ offload
2654
	vmov		q15,q1
2655
	teq		r1,r2
2656
	vld1.32		{q13},[r3]!
2657
	vadd.i32	q12,q12,q8
2658
	INST(0xe2,0x03,0xfa,0xf3)	@ sha256su0 q8,q9
2659
	vmov		q2,q0
2660
	INST(0x68,0x0c,0x02,0xf3)	@ sha256h q0,q1,q12
2661
	INST(0x68,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q12
2662
	INST(0xe6,0x0c,0x64,0xf3)	@ sha256su1 q8,q10,q11
2663
	vld1.32		{q12},[r3]!
2664
	vadd.i32	q13,q13,q9
2665
	INST(0xe4,0x23,0xfa,0xf3)	@ sha256su0 q9,q10
2666
	vmov		q2,q0
2667
	INST(0x6a,0x0c,0x02,0xf3)	@ sha256h q0,q1,q13
2668
	INST(0x6a,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q13
2669
	INST(0xe0,0x2c,0x66,0xf3)	@ sha256su1 q9,q11,q8
2670
	vld1.32		{q13},[r3]!
2671
	vadd.i32	q12,q12,q10
2672
	INST(0xe6,0x43,0xfa,0xf3)	@ sha256su0 q10,q11
2673
	vmov		q2,q0
2674
	INST(0x68,0x0c,0x02,0xf3)	@ sha256h q0,q1,q12
2675
	INST(0x68,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q12
2676
	INST(0xe2,0x4c,0x60,0xf3)	@ sha256su1 q10,q8,q9
2677
	vld1.32		{q12},[r3]!
2678
	vadd.i32	q13,q13,q11
2679
	INST(0xe0,0x63,0xfa,0xf3)	@ sha256su0 q11,q8
2680
	vmov		q2,q0
2681
	INST(0x6a,0x0c,0x02,0xf3)	@ sha256h q0,q1,q13
2682
	INST(0x6a,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q13
2683
	INST(0xe4,0x6c,0x62,0xf3)	@ sha256su1 q11,q9,q10
2684
	vld1.32		{q13},[r3]!
2685
	vadd.i32	q12,q12,q8
2686
	INST(0xe2,0x03,0xfa,0xf3)	@ sha256su0 q8,q9
2687
	vmov		q2,q0
2688
	INST(0x68,0x0c,0x02,0xf3)	@ sha256h q0,q1,q12
2689
	INST(0x68,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q12
2690
	INST(0xe6,0x0c,0x64,0xf3)	@ sha256su1 q8,q10,q11
2691
	vld1.32		{q12},[r3]!
2692
	vadd.i32	q13,q13,q9
2693
	INST(0xe4,0x23,0xfa,0xf3)	@ sha256su0 q9,q10
2694
	vmov		q2,q0
2695
	INST(0x6a,0x0c,0x02,0xf3)	@ sha256h q0,q1,q13
2696
	INST(0x6a,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q13
2697
	INST(0xe0,0x2c,0x66,0xf3)	@ sha256su1 q9,q11,q8
2698
	vld1.32		{q13},[r3]!
2699
	vadd.i32	q12,q12,q10
2700
	INST(0xe6,0x43,0xfa,0xf3)	@ sha256su0 q10,q11
2701
	vmov		q2,q0
2702
	INST(0x68,0x0c,0x02,0xf3)	@ sha256h q0,q1,q12
2703
	INST(0x68,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q12
2704
	INST(0xe2,0x4c,0x60,0xf3)	@ sha256su1 q10,q8,q9
2705
	vld1.32		{q12},[r3]!
2706
	vadd.i32	q13,q13,q11
2707
	INST(0xe0,0x63,0xfa,0xf3)	@ sha256su0 q11,q8
2708
	vmov		q2,q0
2709
	INST(0x6a,0x0c,0x02,0xf3)	@ sha256h q0,q1,q13
2710
	INST(0x6a,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q13
2711
	INST(0xe4,0x6c,0x62,0xf3)	@ sha256su1 q11,q9,q10
2712
	vld1.32		{q13},[r3]!
2713
	vadd.i32	q12,q12,q8
2714
	INST(0xe2,0x03,0xfa,0xf3)	@ sha256su0 q8,q9
2715
	vmov		q2,q0
2716
	INST(0x68,0x0c,0x02,0xf3)	@ sha256h q0,q1,q12
2717
	INST(0x68,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q12
2718
	INST(0xe6,0x0c,0x64,0xf3)	@ sha256su1 q8,q10,q11
2719
	vld1.32		{q12},[r3]!
2720
	vadd.i32	q13,q13,q9
2721
	INST(0xe4,0x23,0xfa,0xf3)	@ sha256su0 q9,q10
2722
	vmov		q2,q0
2723
	INST(0x6a,0x0c,0x02,0xf3)	@ sha256h q0,q1,q13
2724
	INST(0x6a,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q13
2725
	INST(0xe0,0x2c,0x66,0xf3)	@ sha256su1 q9,q11,q8
2726
	vld1.32		{q13},[r3]!
2727
	vadd.i32	q12,q12,q10
2728
	INST(0xe6,0x43,0xfa,0xf3)	@ sha256su0 q10,q11
2729
	vmov		q2,q0
2730
	INST(0x68,0x0c,0x02,0xf3)	@ sha256h q0,q1,q12
2731
	INST(0x68,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q12
2732
	INST(0xe2,0x4c,0x60,0xf3)	@ sha256su1 q10,q8,q9
2733
	vld1.32		{q12},[r3]!
2734
	vadd.i32	q13,q13,q11
2735
	INST(0xe0,0x63,0xfa,0xf3)	@ sha256su0 q11,q8
2736
	vmov		q2,q0
2737
	INST(0x6a,0x0c,0x02,0xf3)	@ sha256h q0,q1,q13
2738
	INST(0x6a,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q13
2739
	INST(0xe4,0x6c,0x62,0xf3)	@ sha256su1 q11,q9,q10
2740
	vld1.32		{q13},[r3]!
2741
	vadd.i32	q12,q12,q8
2742
	vmov		q2,q0
2743
	INST(0x68,0x0c,0x02,0xf3)	@ sha256h q0,q1,q12
2744
	INST(0x68,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q12
2745

2746
	vld1.32		{q12},[r3]!
2747
	vadd.i32	q13,q13,q9
2748
	vmov		q2,q0
2749
	INST(0x6a,0x0c,0x02,0xf3)	@ sha256h q0,q1,q13
2750
	INST(0x6a,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q13
2751

2752
	vld1.32		{q13},[r3]
2753
	vadd.i32	q12,q12,q10
2754
	sub		r3,r3,#256-16	@ rewind
2755
	vmov		q2,q0
2756
	INST(0x68,0x0c,0x02,0xf3)	@ sha256h q0,q1,q12
2757
	INST(0x68,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q12
2758

2759
	vadd.i32	q13,q13,q11
2760
	vmov		q2,q0
2761
	INST(0x6a,0x0c,0x02,0xf3)	@ sha256h q0,q1,q13
2762
	INST(0x6a,0x2c,0x14,0xf3)	@ sha256h2 q1,q2,q13
2763

2764
	vadd.i32	q0,q0,q14
2765
	vadd.i32	q1,q1,q15
2766
	it		ne
2767
	bne		.Loop_v8
2768

2769
	vst1.32		{q0,q1},[r0]
2770

2771
	bx	lr		@ bx lr
2772
.size	zfs_sha256_block_armv8,.-zfs_sha256_block_armv8
2773

2774
#endif // #if __ARM_ARCH__ >= 7
2775
#endif // #if defined(__arm__)
2776

2777
Product

Resources

Company
