CoCalc -- aesp8-ppc.pl

GitHub Repository: torvalds/linux
Path: blob/master/arch/powerpc/crypto/aesp8-ppc.pl
²⁶⁴²⁴ views
1
#! /usr/bin/env perl
2
# SPDX-License-Identifier: GPL-2.0
3

4
# This code is taken from CRYPTOGAMs[1] and is included here using the option
5
# in the license to distribute the code under the GPL. Therefore this program
6
# is free software; you can redistribute it and/or modify it under the terms of
7
# the GNU General Public License version 2 as published by the Free Software
8
# Foundation.
9
#
10
# [1] https://www.openssl.org/~appro/cryptogams/
11

12
# Copyright (c) 2006-2017, CRYPTOGAMS by <[email protected]>
13
# All rights reserved.
14
#
15
# Redistribution and use in source and binary forms, with or without
16
# modification, are permitted provided that the following conditions
17
# are met:
18
#
19
#       * Redistributions of source code must retain copyright notices,
20
#         this list of conditions and the following disclaimer.
21
#
22
#       * Redistributions in binary form must reproduce the above
23
#         copyright notice, this list of conditions and the following
24
#         disclaimer in the documentation and/or other materials
25
#         provided with the distribution.
26
#
27
#       * Neither the name of the CRYPTOGAMS nor the names of its
28
#         copyright holder and contributors may be used to endorse or
29
#         promote products derived from this software without specific
30
#         prior written permission.
31
#
32
# ALTERNATIVELY, provided that this notice is retained in full, this
33
# product may be distributed under the terms of the GNU General Public
34
# License (GPL), in which case the provisions of the GPL apply INSTEAD OF
35
# those given above.
36
#
37
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
38
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
39
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
40
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
41
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
42
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
43
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
44
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
45
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
46
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
47
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
48

49
# ====================================================================
50
# Written by Andy Polyakov <[email protected]> for the OpenSSL
51
# project. The module is, however, dual licensed under OpenSSL and
52
# CRYPTOGAMS licenses depending on where you obtain it. For further
53
# details see https://www.openssl.org/~appro/cryptogams/.
54
# ====================================================================
55
#
56
# This module implements support for AES instructions as per PowerISA
57
# specification version 2.07, first implemented by POWER8 processor.
58
# The module is endian-agnostic in sense that it supports both big-
59
# and little-endian cases. Data alignment in parallelizable modes is
60
# handled with VSX loads and stores, which implies MSR.VSX flag being
61
# set. It should also be noted that ISA specification doesn't prohibit
62
# alignment exceptions for these instructions on page boundaries.
63
# Initially alignment was handled in pure AltiVec/VMX way [when data
64
# is aligned programmatically, which in turn guarantees exception-
65
# free execution], but it turned to hamper performance when vcipher
66
# instructions are interleaved. It's reckoned that eventual
67
# misalignment penalties at page boundaries are in average lower
68
# than additional overhead in pure AltiVec approach.
69
#
70
# May 2016
71
#
72
# Add XTS subroutine, 9x on little- and 12x improvement on big-endian
73
# systems were measured.
74
#
75
######################################################################
76
# Current large-block performance in cycles per byte processed with
77
# 128-bit key (less is better).
78
#
79
#		CBC en-/decrypt	CTR	XTS
80
# POWER8[le]	3.96/0.72	0.74	1.1
81
# POWER8[be]	3.75/0.65	0.66	1.0
82

83
$flavour = shift;
84

85
if ($flavour =~ /64/) {
86
	$SIZE_T	=8;
87
	$LRSAVE	=2*$SIZE_T;
88
	$STU	="stdu";
89
	$POP	="ld";
90
	$PUSH	="std";
91
	$UCMP	="cmpld";
92
	$SHL	="sldi";
93
} elsif ($flavour =~ /32/) {
94
	$SIZE_T	=4;
95
	$LRSAVE	=$SIZE_T;
96
	$STU	="stwu";
97
	$POP	="lwz";
98
	$PUSH	="stw";
99
	$UCMP	="cmplw";
100
	$SHL	="slwi";
101
} else { die "nonsense $flavour"; }
102

103
$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
104

105
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
106
( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
107
( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
108
die "can't locate ppc-xlate.pl";
109

110
open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
111

112
$FRAME=8*$SIZE_T;
113
$prefix="aes_p8";
114

115
$sp="r1";
116
$vrsave="r12";
117

118
#########################################################################
119
{{{	# Key setup procedures						#
120
my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8));
121
my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));
122
my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11));
123

124
$code.=<<___;
125
.machine	"any"
126

127
.text
128

129
.align	7
130
rcon:
131
.long	0x01000000, 0x01000000, 0x01000000, 0x01000000	?rev
132
.long	0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000	?rev
133
.long	0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c	?rev
134
.long	0,0,0,0						?asis
135
.long	0x0f102132, 0x43546576, 0x8798a9ba, 0xcbdcedfe
136
Lconsts:
137
	mflr	r0
138
	bcl	20,31,\$+4
139
	mflr	$ptr	 #vvvvv "distance between . and rcon
140
	addi	$ptr,$ptr,-0x58
141
	mtlr	r0
142
	blr
143
	.long	0
144
	.byte	0,12,0x14,0,0,0,0,0
145
.asciz	"AES for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
146

147
.globl	.${prefix}_set_encrypt_key
148
Lset_encrypt_key:
149
	mflr		r11
150
	$PUSH		r11,$LRSAVE($sp)
151

152
	li		$ptr,-1
153
	${UCMP}i	$inp,0
154
	beq-		Lenc_key_abort		# if ($inp==0) return -1;
155
	${UCMP}i	$out,0
156
	beq-		Lenc_key_abort		# if ($out==0) return -1;
157
	li		$ptr,-2
158
	cmpwi		$bits,128
159
	blt-		Lenc_key_abort
160
	cmpwi		$bits,256
161
	bgt-		Lenc_key_abort
162
	andi.		r0,$bits,0x3f
163
	bne-		Lenc_key_abort
164

165
	lis		r0,0xfff0
166
	mfspr		$vrsave,256
167
	mtspr		256,r0
168

169
	bl		Lconsts
170
	mtlr		r11
171

172
	neg		r9,$inp
173
	lvx		$in0,0,$inp
174
	addi		$inp,$inp,15		# 15 is not typo
175
	lvsr		$key,0,r9		# borrow $key
176
	li		r8,0x20
177
	cmpwi		$bits,192
178
	lvx		$in1,0,$inp
179
	le?vspltisb	$mask,0x0f		# borrow $mask
180
	lvx		$rcon,0,$ptr
181
	le?vxor		$key,$key,$mask		# adjust for byte swap
182
	lvx		$mask,r8,$ptr
183
	addi		$ptr,$ptr,0x10
184
	vperm		$in0,$in0,$in1,$key	# align [and byte swap in LE]
185
	li		$cnt,8
186
	vxor		$zero,$zero,$zero
187
	mtctr		$cnt
188

189
	?lvsr		$outperm,0,$out
190
	vspltisb	$outmask,-1
191
	lvx		$outhead,0,$out
192
	?vperm		$outmask,$zero,$outmask,$outperm
193

194
	blt		Loop128
195
	addi		$inp,$inp,8
196
	beq		L192
197
	addi		$inp,$inp,8
198
	b		L256
199

200
.align	4
201
Loop128:
202
	vperm		$key,$in0,$in0,$mask	# rotate-n-splat
203
	vsldoi		$tmp,$zero,$in0,12	# >>32
204
	 vperm		$outtail,$in0,$in0,$outperm	# rotate
205
	 vsel		$stage,$outhead,$outtail,$outmask
206
	 vmr		$outhead,$outtail
207
	vcipherlast	$key,$key,$rcon
208
	 stvx		$stage,0,$out
209
	 addi		$out,$out,16
210

211
	vxor		$in0,$in0,$tmp
212
	vsldoi		$tmp,$zero,$tmp,12	# >>32
213
	vxor		$in0,$in0,$tmp
214
	vsldoi		$tmp,$zero,$tmp,12	# >>32
215
	vxor		$in0,$in0,$tmp
216
	 vadduwm	$rcon,$rcon,$rcon
217
	vxor		$in0,$in0,$key
218
	bdnz		Loop128
219

220
	lvx		$rcon,0,$ptr		# last two round keys
221

222
	vperm		$key,$in0,$in0,$mask	# rotate-n-splat
223
	vsldoi		$tmp,$zero,$in0,12	# >>32
224
	 vperm		$outtail,$in0,$in0,$outperm	# rotate
225
	 vsel		$stage,$outhead,$outtail,$outmask
226
	 vmr		$outhead,$outtail
227
	vcipherlast	$key,$key,$rcon
228
	 stvx		$stage,0,$out
229
	 addi		$out,$out,16
230

231
	vxor		$in0,$in0,$tmp
232
	vsldoi		$tmp,$zero,$tmp,12	# >>32
233
	vxor		$in0,$in0,$tmp
234
	vsldoi		$tmp,$zero,$tmp,12	# >>32
235
	vxor		$in0,$in0,$tmp
236
	 vadduwm	$rcon,$rcon,$rcon
237
	vxor		$in0,$in0,$key
238

239
	vperm		$key,$in0,$in0,$mask	# rotate-n-splat
240
	vsldoi		$tmp,$zero,$in0,12	# >>32
241
	 vperm		$outtail,$in0,$in0,$outperm	# rotate
242
	 vsel		$stage,$outhead,$outtail,$outmask
243
	 vmr		$outhead,$outtail
244
	vcipherlast	$key,$key,$rcon
245
	 stvx		$stage,0,$out
246
	 addi		$out,$out,16
247

248
	vxor		$in0,$in0,$tmp
249
	vsldoi		$tmp,$zero,$tmp,12	# >>32
250
	vxor		$in0,$in0,$tmp
251
	vsldoi		$tmp,$zero,$tmp,12	# >>32
252
	vxor		$in0,$in0,$tmp
253
	vxor		$in0,$in0,$key
254
	 vperm		$outtail,$in0,$in0,$outperm	# rotate
255
	 vsel		$stage,$outhead,$outtail,$outmask
256
	 vmr		$outhead,$outtail
257
	 stvx		$stage,0,$out
258

259
	addi		$inp,$out,15		# 15 is not typo
260
	addi		$out,$out,0x50
261

262
	li		$rounds,10
263
	b		Ldone
264

265
.align	4
266
L192:
267
	lvx		$tmp,0,$inp
268
	li		$cnt,4
269
	 vperm		$outtail,$in0,$in0,$outperm	# rotate
270
	 vsel		$stage,$outhead,$outtail,$outmask
271
	 vmr		$outhead,$outtail
272
	 stvx		$stage,0,$out
273
	 addi		$out,$out,16
274
	vperm		$in1,$in1,$tmp,$key	# align [and byte swap in LE]
275
	vspltisb	$key,8			# borrow $key
276
	mtctr		$cnt
277
	vsububm		$mask,$mask,$key	# adjust the mask
278

279
Loop192:
280
	vperm		$key,$in1,$in1,$mask	# roate-n-splat
281
	vsldoi		$tmp,$zero,$in0,12	# >>32
282
	vcipherlast	$key,$key,$rcon
283

284
	vxor		$in0,$in0,$tmp
285
	vsldoi		$tmp,$zero,$tmp,12	# >>32
286
	vxor		$in0,$in0,$tmp
287
	vsldoi		$tmp,$zero,$tmp,12	# >>32
288
	vxor		$in0,$in0,$tmp
289

290
	 vsldoi		$stage,$zero,$in1,8
291
	vspltw		$tmp,$in0,3
292
	vxor		$tmp,$tmp,$in1
293
	vsldoi		$in1,$zero,$in1,12	# >>32
294
	 vadduwm	$rcon,$rcon,$rcon
295
	vxor		$in1,$in1,$tmp
296
	vxor		$in0,$in0,$key
297
	vxor		$in1,$in1,$key
298
	 vsldoi		$stage,$stage,$in0,8
299

300
	vperm		$key,$in1,$in1,$mask	# rotate-n-splat
301
	vsldoi		$tmp,$zero,$in0,12	# >>32
302
	 vperm		$outtail,$stage,$stage,$outperm	# rotate
303
	 vsel		$stage,$outhead,$outtail,$outmask
304
	 vmr		$outhead,$outtail
305
	vcipherlast	$key,$key,$rcon
306
	 stvx		$stage,0,$out
307
	 addi		$out,$out,16
308

309
	 vsldoi		$stage,$in0,$in1,8
310
	vxor		$in0,$in0,$tmp
311
	vsldoi		$tmp,$zero,$tmp,12	# >>32
312
	 vperm		$outtail,$stage,$stage,$outperm	# rotate
313
	 vsel		$stage,$outhead,$outtail,$outmask
314
	 vmr		$outhead,$outtail
315
	vxor		$in0,$in0,$tmp
316
	vsldoi		$tmp,$zero,$tmp,12	# >>32
317
	vxor		$in0,$in0,$tmp
318
	 stvx		$stage,0,$out
319
	 addi		$out,$out,16
320

321
	vspltw		$tmp,$in0,3
322
	vxor		$tmp,$tmp,$in1
323
	vsldoi		$in1,$zero,$in1,12	# >>32
324
	 vadduwm	$rcon,$rcon,$rcon
325
	vxor		$in1,$in1,$tmp
326
	vxor		$in0,$in0,$key
327
	vxor		$in1,$in1,$key
328
	 vperm		$outtail,$in0,$in0,$outperm	# rotate
329
	 vsel		$stage,$outhead,$outtail,$outmask
330
	 vmr		$outhead,$outtail
331
	 stvx		$stage,0,$out
332
	 addi		$inp,$out,15		# 15 is not typo
333
	 addi		$out,$out,16
334
	bdnz		Loop192
335

336
	li		$rounds,12
337
	addi		$out,$out,0x20
338
	b		Ldone
339

340
.align	4
341
L256:
342
	lvx		$tmp,0,$inp
343
	li		$cnt,7
344
	li		$rounds,14
345
	 vperm		$outtail,$in0,$in0,$outperm	# rotate
346
	 vsel		$stage,$outhead,$outtail,$outmask
347
	 vmr		$outhead,$outtail
348
	 stvx		$stage,0,$out
349
	 addi		$out,$out,16
350
	vperm		$in1,$in1,$tmp,$key	# align [and byte swap in LE]
351
	mtctr		$cnt
352

353
Loop256:
354
	vperm		$key,$in1,$in1,$mask	# rotate-n-splat
355
	vsldoi		$tmp,$zero,$in0,12	# >>32
356
	 vperm		$outtail,$in1,$in1,$outperm	# rotate
357
	 vsel		$stage,$outhead,$outtail,$outmask
358
	 vmr		$outhead,$outtail
359
	vcipherlast	$key,$key,$rcon
360
	 stvx		$stage,0,$out
361
	 addi		$out,$out,16
362

363
	vxor		$in0,$in0,$tmp
364
	vsldoi		$tmp,$zero,$tmp,12	# >>32
365
	vxor		$in0,$in0,$tmp
366
	vsldoi		$tmp,$zero,$tmp,12	# >>32
367
	vxor		$in0,$in0,$tmp
368
	 vadduwm	$rcon,$rcon,$rcon
369
	vxor		$in0,$in0,$key
370
	 vperm		$outtail,$in0,$in0,$outperm	# rotate
371
	 vsel		$stage,$outhead,$outtail,$outmask
372
	 vmr		$outhead,$outtail
373
	 stvx		$stage,0,$out
374
	 addi		$inp,$out,15		# 15 is not typo
375
	 addi		$out,$out,16
376
	bdz		Ldone
377

378
	vspltw		$key,$in0,3		# just splat
379
	vsldoi		$tmp,$zero,$in1,12	# >>32
380
	vsbox		$key,$key
381

382
	vxor		$in1,$in1,$tmp
383
	vsldoi		$tmp,$zero,$tmp,12	# >>32
384
	vxor		$in1,$in1,$tmp
385
	vsldoi		$tmp,$zero,$tmp,12	# >>32
386
	vxor		$in1,$in1,$tmp
387

388
	vxor		$in1,$in1,$key
389
	b		Loop256
390

391
.align	4
392
Ldone:
393
	lvx		$in1,0,$inp		# redundant in aligned case
394
	vsel		$in1,$outhead,$in1,$outmask
395
	stvx		$in1,0,$inp
396
	li		$ptr,0
397
	mtspr		256,$vrsave
398
	stw		$rounds,0($out)
399

400
Lenc_key_abort:
401
	mr		r3,$ptr
402
	blr
403
	.long		0
404
	.byte		0,12,0x14,1,0,0,3,0
405
	.long		0
406
.size	.${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
407

408
.globl	.${prefix}_set_decrypt_key
409
	$STU		$sp,-$FRAME($sp)
410
	mflr		r10
411
	$PUSH		r10,$FRAME+$LRSAVE($sp)
412
	bl		Lset_encrypt_key
413
	mtlr		r10
414

415
	cmpwi		r3,0
416
	bne-		Ldec_key_abort
417

418
	slwi		$cnt,$rounds,4
419
	subi		$inp,$out,240		# first round key
420
	srwi		$rounds,$rounds,1
421
	add		$out,$inp,$cnt		# last round key
422
	mtctr		$rounds
423

424
Ldeckey:
425
	lwz		r0, 0($inp)
426
	lwz		r6, 4($inp)
427
	lwz		r7, 8($inp)
428
	lwz		r8, 12($inp)
429
	addi		$inp,$inp,16
430
	lwz		r9, 0($out)
431
	lwz		r10,4($out)
432
	lwz		r11,8($out)
433
	lwz		r12,12($out)
434
	stw		r0, 0($out)
435
	stw		r6, 4($out)
436
	stw		r7, 8($out)
437
	stw		r8, 12($out)
438
	subi		$out,$out,16
439
	stw		r9, -16($inp)
440
	stw		r10,-12($inp)
441
	stw		r11,-8($inp)
442
	stw		r12,-4($inp)
443
	bdnz		Ldeckey
444

445
	xor		r3,r3,r3		# return value
446
Ldec_key_abort:
447
	addi		$sp,$sp,$FRAME
448
	blr
449
	.long		0
450
	.byte		0,12,4,1,0x80,0,3,0
451
	.long		0
452
.size	.${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
453
___
454
}}}
455
#########################################################################
456
{{{	# Single block en- and decrypt procedures			#
457
sub gen_block () {
458
my $dir = shift;
459
my $n   = $dir eq "de" ? "n" : "";
460
my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7));
461

462
$code.=<<___;
463
.globl	.${prefix}_${dir}crypt
464
	lwz		$rounds,240($key)
465
	lis		r0,0xfc00
466
	mfspr		$vrsave,256
467
	li		$idx,15			# 15 is not typo
468
	mtspr		256,r0
469

470
	lvx		v0,0,$inp
471
	neg		r11,$out
472
	lvx		v1,$idx,$inp
473
	lvsl		v2,0,$inp		# inpperm
474
	le?vspltisb	v4,0x0f
475
	?lvsl		v3,0,r11		# outperm
476
	le?vxor		v2,v2,v4
477
	li		$idx,16
478
	vperm		v0,v0,v1,v2		# align [and byte swap in LE]
479
	lvx		v1,0,$key
480
	?lvsl		v5,0,$key		# keyperm
481
	srwi		$rounds,$rounds,1
482
	lvx		v2,$idx,$key
483
	addi		$idx,$idx,16
484
	subi		$rounds,$rounds,1
485
	?vperm		v1,v1,v2,v5		# align round key
486

487
	vxor		v0,v0,v1
488
	lvx		v1,$idx,$key
489
	addi		$idx,$idx,16
490
	mtctr		$rounds
491

492
Loop_${dir}c:
493
	?vperm		v2,v2,v1,v5
494
	v${n}cipher	v0,v0,v2
495
	lvx		v2,$idx,$key
496
	addi		$idx,$idx,16
497
	?vperm		v1,v1,v2,v5
498
	v${n}cipher	v0,v0,v1
499
	lvx		v1,$idx,$key
500
	addi		$idx,$idx,16
501
	bdnz		Loop_${dir}c
502

503
	?vperm		v2,v2,v1,v5
504
	v${n}cipher	v0,v0,v2
505
	lvx		v2,$idx,$key
506
	?vperm		v1,v1,v2,v5
507
	v${n}cipherlast	v0,v0,v1
508

509
	vspltisb	v2,-1
510
	vxor		v1,v1,v1
511
	li		$idx,15			# 15 is not typo
512
	?vperm		v2,v1,v2,v3		# outmask
513
	le?vxor		v3,v3,v4
514
	lvx		v1,0,$out		# outhead
515
	vperm		v0,v0,v0,v3		# rotate [and byte swap in LE]
516
	vsel		v1,v1,v0,v2
517
	lvx		v4,$idx,$out
518
	stvx		v1,0,$out
519
	vsel		v0,v0,v4,v2
520
	stvx		v0,$idx,$out
521

522
	mtspr		256,$vrsave
523
	blr
524
	.long		0
525
	.byte		0,12,0x14,0,0,0,3,0
526
	.long		0
527
.size	.${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
528
___
529
}
530
&gen_block("en");
531
&gen_block("de");
532
}}}
533
#########################################################################
534
{{{	# CBC en- and decrypt procedures				#
535
my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10));
536
my ($rndkey0,$rndkey1,$inout,$tmp)=		map("v$_",(0..3));
537
my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=
538
						map("v$_",(4..10));
539
$code.=<<___;
540
.globl	.${prefix}_cbc_encrypt
541
	${UCMP}i	$len,16
542
	bltlr-
543

544
	cmpwi		$enc,0			# test direction
545
	lis		r0,0xffe0
546
	mfspr		$vrsave,256
547
	mtspr		256,r0
548

549
	li		$idx,15
550
	vxor		$rndkey0,$rndkey0,$rndkey0
551
	le?vspltisb	$tmp,0x0f
552

553
	lvx		$ivec,0,$ivp		# load [unaligned] iv
554
	lvsl		$inpperm,0,$ivp
555
	lvx		$inptail,$idx,$ivp
556
	le?vxor		$inpperm,$inpperm,$tmp
557
	vperm		$ivec,$ivec,$inptail,$inpperm
558

559
	neg		r11,$inp
560
	?lvsl		$keyperm,0,$key		# prepare for unaligned key
561
	lwz		$rounds,240($key)
562

563
	lvsr		$inpperm,0,r11		# prepare for unaligned load
564
	lvx		$inptail,0,$inp
565
	addi		$inp,$inp,15		# 15 is not typo
566
	le?vxor		$inpperm,$inpperm,$tmp
567

568
	?lvsr		$outperm,0,$out		# prepare for unaligned store
569
	vspltisb	$outmask,-1
570
	lvx		$outhead,0,$out
571
	?vperm		$outmask,$rndkey0,$outmask,$outperm
572
	le?vxor		$outperm,$outperm,$tmp
573

574
	srwi		$rounds,$rounds,1
575
	li		$idx,16
576
	subi		$rounds,$rounds,1
577
	beq		Lcbc_dec
578

579
Lcbc_enc:
580
	vmr		$inout,$inptail
581
	lvx		$inptail,0,$inp
582
	addi		$inp,$inp,16
583
	mtctr		$rounds
584
	subi		$len,$len,16		# len-=16
585

586
	lvx		$rndkey0,0,$key
587
	 vperm		$inout,$inout,$inptail,$inpperm
588
	lvx		$rndkey1,$idx,$key
589
	addi		$idx,$idx,16
590
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
591
	vxor		$inout,$inout,$rndkey0
592
	lvx		$rndkey0,$idx,$key
593
	addi		$idx,$idx,16
594
	vxor		$inout,$inout,$ivec
595

596
Loop_cbc_enc:
597
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
598
	vcipher		$inout,$inout,$rndkey1
599
	lvx		$rndkey1,$idx,$key
600
	addi		$idx,$idx,16
601
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
602
	vcipher		$inout,$inout,$rndkey0
603
	lvx		$rndkey0,$idx,$key
604
	addi		$idx,$idx,16
605
	bdnz		Loop_cbc_enc
606

607
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
608
	vcipher		$inout,$inout,$rndkey1
609
	lvx		$rndkey1,$idx,$key
610
	li		$idx,16
611
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
612
	vcipherlast	$ivec,$inout,$rndkey0
613
	${UCMP}i	$len,16
614

615
	vperm		$tmp,$ivec,$ivec,$outperm
616
	vsel		$inout,$outhead,$tmp,$outmask
617
	vmr		$outhead,$tmp
618
	stvx		$inout,0,$out
619
	addi		$out,$out,16
620
	bge		Lcbc_enc
621

622
	b		Lcbc_done
623

624
.align	4
625
Lcbc_dec:
626
	${UCMP}i	$len,128
627
	bge		_aesp8_cbc_decrypt8x
628
	vmr		$tmp,$inptail
629
	lvx		$inptail,0,$inp
630
	addi		$inp,$inp,16
631
	mtctr		$rounds
632
	subi		$len,$len,16		# len-=16
633

634
	lvx		$rndkey0,0,$key
635
	 vperm		$tmp,$tmp,$inptail,$inpperm
636
	lvx		$rndkey1,$idx,$key
637
	addi		$idx,$idx,16
638
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
639
	vxor		$inout,$tmp,$rndkey0
640
	lvx		$rndkey0,$idx,$key
641
	addi		$idx,$idx,16
642

643
Loop_cbc_dec:
644
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
645
	vncipher	$inout,$inout,$rndkey1
646
	lvx		$rndkey1,$idx,$key
647
	addi		$idx,$idx,16
648
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
649
	vncipher	$inout,$inout,$rndkey0
650
	lvx		$rndkey0,$idx,$key
651
	addi		$idx,$idx,16
652
	bdnz		Loop_cbc_dec
653

654
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
655
	vncipher	$inout,$inout,$rndkey1
656
	lvx		$rndkey1,$idx,$key
657
	li		$idx,16
658
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
659
	vncipherlast	$inout,$inout,$rndkey0
660
	${UCMP}i	$len,16
661

662
	vxor		$inout,$inout,$ivec
663
	vmr		$ivec,$tmp
664
	vperm		$tmp,$inout,$inout,$outperm
665
	vsel		$inout,$outhead,$tmp,$outmask
666
	vmr		$outhead,$tmp
667
	stvx		$inout,0,$out
668
	addi		$out,$out,16
669
	bge		Lcbc_dec
670

671
Lcbc_done:
672
	addi		$out,$out,-1
673
	lvx		$inout,0,$out		# redundant in aligned case
674
	vsel		$inout,$outhead,$inout,$outmask
675
	stvx		$inout,0,$out
676

677
	neg		$enc,$ivp		# write [unaligned] iv
678
	li		$idx,15			# 15 is not typo
679
	vxor		$rndkey0,$rndkey0,$rndkey0
680
	vspltisb	$outmask,-1
681
	le?vspltisb	$tmp,0x0f
682
	?lvsl		$outperm,0,$enc
683
	?vperm		$outmask,$rndkey0,$outmask,$outperm
684
	le?vxor		$outperm,$outperm,$tmp
685
	lvx		$outhead,0,$ivp
686
	vperm		$ivec,$ivec,$ivec,$outperm
687
	vsel		$inout,$outhead,$ivec,$outmask
688
	lvx		$inptail,$idx,$ivp
689
	stvx		$inout,0,$ivp
690
	vsel		$inout,$ivec,$inptail,$outmask
691
	stvx		$inout,$idx,$ivp
692

693
	mtspr		256,$vrsave
694
	blr
695
	.long		0
696
	.byte		0,12,0x14,0,0,0,6,0
697
	.long		0
698
___
699
#########################################################################
700
{{	# Optimized CBC decrypt procedure				#
701
my $key_="r11";
702
my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
703
my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
704
my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21));
705
my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
706
			# v26-v31 last 6 round keys
707
my ($tmp,$keyperm)=($in3,$in4);	# aliases with "caller", redundant assignment
708

709
$code.=<<___;
710
.align	5
711
_aesp8_cbc_decrypt8x:
712
	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
713
	li		r10,`$FRAME+8*16+15`
714
	li		r11,`$FRAME+8*16+31`
715
	stvx		v20,r10,$sp		# ABI says so
716
	addi		r10,r10,32
717
	stvx		v21,r11,$sp
718
	addi		r11,r11,32
719
	stvx		v22,r10,$sp
720
	addi		r10,r10,32
721
	stvx		v23,r11,$sp
722
	addi		r11,r11,32
723
	stvx		v24,r10,$sp
724
	addi		r10,r10,32
725
	stvx		v25,r11,$sp
726
	addi		r11,r11,32
727
	stvx		v26,r10,$sp
728
	addi		r10,r10,32
729
	stvx		v27,r11,$sp
730
	addi		r11,r11,32
731
	stvx		v28,r10,$sp
732
	addi		r10,r10,32
733
	stvx		v29,r11,$sp
734
	addi		r11,r11,32
735
	stvx		v30,r10,$sp
736
	stvx		v31,r11,$sp
737
	li		r0,-1
738
	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
739
	li		$x10,0x10
740
	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
741
	li		$x20,0x20
742
	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
743
	li		$x30,0x30
744
	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
745
	li		$x40,0x40
746
	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
747
	li		$x50,0x50
748
	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
749
	li		$x60,0x60
750
	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
751
	li		$x70,0x70
752
	mtspr		256,r0
753

754
	subi		$rounds,$rounds,3	# -4 in total
755
	subi		$len,$len,128		# bias
756

757
	lvx		$rndkey0,$x00,$key	# load key schedule
758
	lvx		v30,$x10,$key
759
	addi		$key,$key,0x20
760
	lvx		v31,$x00,$key
761
	?vperm		$rndkey0,$rndkey0,v30,$keyperm
762
	addi		$key_,$sp,$FRAME+15
763
	mtctr		$rounds
764

765
Load_cbc_dec_key:
766
	?vperm		v24,v30,v31,$keyperm
767
	lvx		v30,$x10,$key
768
	addi		$key,$key,0x20
769
	stvx		v24,$x00,$key_		# off-load round[1]
770
	?vperm		v25,v31,v30,$keyperm
771
	lvx		v31,$x00,$key
772
	stvx		v25,$x10,$key_		# off-load round[2]
773
	addi		$key_,$key_,0x20
774
	bdnz		Load_cbc_dec_key
775

776
	lvx		v26,$x10,$key
777
	?vperm		v24,v30,v31,$keyperm
778
	lvx		v27,$x20,$key
779
	stvx		v24,$x00,$key_		# off-load round[3]
780
	?vperm		v25,v31,v26,$keyperm
781
	lvx		v28,$x30,$key
782
	stvx		v25,$x10,$key_		# off-load round[4]
783
	addi		$key_,$sp,$FRAME+15	# rewind $key_
784
	?vperm		v26,v26,v27,$keyperm
785
	lvx		v29,$x40,$key
786
	?vperm		v27,v27,v28,$keyperm
787
	lvx		v30,$x50,$key
788
	?vperm		v28,v28,v29,$keyperm
789
	lvx		v31,$x60,$key
790
	?vperm		v29,v29,v30,$keyperm
791
	lvx		$out0,$x70,$key		# borrow $out0
792
	?vperm		v30,v30,v31,$keyperm
793
	lvx		v24,$x00,$key_		# pre-load round[1]
794
	?vperm		v31,v31,$out0,$keyperm
795
	lvx		v25,$x10,$key_		# pre-load round[2]
796

797
	#lvx		$inptail,0,$inp		# "caller" already did this
798
	#addi		$inp,$inp,15		# 15 is not typo
799
	subi		$inp,$inp,15		# undo "caller"
800

801
	 le?li		$idx,8
802
	lvx_u		$in0,$x00,$inp		# load first 8 "words"
803
	 le?lvsl	$inpperm,0,$idx
804
	 le?vspltisb	$tmp,0x0f
805
	lvx_u		$in1,$x10,$inp
806
	 le?vxor	$inpperm,$inpperm,$tmp	# transform for lvx_u/stvx_u
807
	lvx_u		$in2,$x20,$inp
808
	 le?vperm	$in0,$in0,$in0,$inpperm
809
	lvx_u		$in3,$x30,$inp
810
	 le?vperm	$in1,$in1,$in1,$inpperm
811
	lvx_u		$in4,$x40,$inp
812
	 le?vperm	$in2,$in2,$in2,$inpperm
813
	vxor		$out0,$in0,$rndkey0
814
	lvx_u		$in5,$x50,$inp
815
	 le?vperm	$in3,$in3,$in3,$inpperm
816
	vxor		$out1,$in1,$rndkey0
817
	lvx_u		$in6,$x60,$inp
818
	 le?vperm	$in4,$in4,$in4,$inpperm
819
	vxor		$out2,$in2,$rndkey0
820
	lvx_u		$in7,$x70,$inp
821
	addi		$inp,$inp,0x80
822
	 le?vperm	$in5,$in5,$in5,$inpperm
823
	vxor		$out3,$in3,$rndkey0
824
	 le?vperm	$in6,$in6,$in6,$inpperm
825
	vxor		$out4,$in4,$rndkey0
826
	 le?vperm	$in7,$in7,$in7,$inpperm
827
	vxor		$out5,$in5,$rndkey0
828
	vxor		$out6,$in6,$rndkey0
829
	vxor		$out7,$in7,$rndkey0
830

831
	mtctr		$rounds
832
	b		Loop_cbc_dec8x
833
.align	5
834
Loop_cbc_dec8x:
835
	vncipher	$out0,$out0,v24
836
	vncipher	$out1,$out1,v24
837
	vncipher	$out2,$out2,v24
838
	vncipher	$out3,$out3,v24
839
	vncipher	$out4,$out4,v24
840
	vncipher	$out5,$out5,v24
841
	vncipher	$out6,$out6,v24
842
	vncipher	$out7,$out7,v24
843
	lvx		v24,$x20,$key_		# round[3]
844
	addi		$key_,$key_,0x20
845

846
	vncipher	$out0,$out0,v25
847
	vncipher	$out1,$out1,v25
848
	vncipher	$out2,$out2,v25
849
	vncipher	$out3,$out3,v25
850
	vncipher	$out4,$out4,v25
851
	vncipher	$out5,$out5,v25
852
	vncipher	$out6,$out6,v25
853
	vncipher	$out7,$out7,v25
854
	lvx		v25,$x10,$key_		# round[4]
855
	bdnz		Loop_cbc_dec8x
856

857
	subic		$len,$len,128		# $len-=128
858
	vncipher	$out0,$out0,v24
859
	vncipher	$out1,$out1,v24
860
	vncipher	$out2,$out2,v24
861
	vncipher	$out3,$out3,v24
862
	vncipher	$out4,$out4,v24
863
	vncipher	$out5,$out5,v24
864
	vncipher	$out6,$out6,v24
865
	vncipher	$out7,$out7,v24
866

867
	subfe.		r0,r0,r0		# borrow?-1:0
868
	vncipher	$out0,$out0,v25
869
	vncipher	$out1,$out1,v25
870
	vncipher	$out2,$out2,v25
871
	vncipher	$out3,$out3,v25
872
	vncipher	$out4,$out4,v25
873
	vncipher	$out5,$out5,v25
874
	vncipher	$out6,$out6,v25
875
	vncipher	$out7,$out7,v25
876

877
	and		r0,r0,$len
878
	vncipher	$out0,$out0,v26
879
	vncipher	$out1,$out1,v26
880
	vncipher	$out2,$out2,v26
881
	vncipher	$out3,$out3,v26
882
	vncipher	$out4,$out4,v26
883
	vncipher	$out5,$out5,v26
884
	vncipher	$out6,$out6,v26
885
	vncipher	$out7,$out7,v26
886

887
	add		$inp,$inp,r0		# $inp is adjusted in such
888
						# way that at exit from the
889
						# loop inX-in7 are loaded
890
						# with last "words"
891
	vncipher	$out0,$out0,v27
892
	vncipher	$out1,$out1,v27
893
	vncipher	$out2,$out2,v27
894
	vncipher	$out3,$out3,v27
895
	vncipher	$out4,$out4,v27
896
	vncipher	$out5,$out5,v27
897
	vncipher	$out6,$out6,v27
898
	vncipher	$out7,$out7,v27
899

900
	addi		$key_,$sp,$FRAME+15	# rewind $key_
901
	vncipher	$out0,$out0,v28
902
	vncipher	$out1,$out1,v28
903
	vncipher	$out2,$out2,v28
904
	vncipher	$out3,$out3,v28
905
	vncipher	$out4,$out4,v28
906
	vncipher	$out5,$out5,v28
907
	vncipher	$out6,$out6,v28
908
	vncipher	$out7,$out7,v28
909
	lvx		v24,$x00,$key_		# re-pre-load round[1]
910

911
	vncipher	$out0,$out0,v29
912
	vncipher	$out1,$out1,v29
913
	vncipher	$out2,$out2,v29
914
	vncipher	$out3,$out3,v29
915
	vncipher	$out4,$out4,v29
916
	vncipher	$out5,$out5,v29
917
	vncipher	$out6,$out6,v29
918
	vncipher	$out7,$out7,v29
919
	lvx		v25,$x10,$key_		# re-pre-load round[2]
920

921
	vncipher	$out0,$out0,v30
922
	 vxor		$ivec,$ivec,v31		# xor with last round key
923
	vncipher	$out1,$out1,v30
924
	 vxor		$in0,$in0,v31
925
	vncipher	$out2,$out2,v30
926
	 vxor		$in1,$in1,v31
927
	vncipher	$out3,$out3,v30
928
	 vxor		$in2,$in2,v31
929
	vncipher	$out4,$out4,v30
930
	 vxor		$in3,$in3,v31
931
	vncipher	$out5,$out5,v30
932
	 vxor		$in4,$in4,v31
933
	vncipher	$out6,$out6,v30
934
	 vxor		$in5,$in5,v31
935
	vncipher	$out7,$out7,v30
936
	 vxor		$in6,$in6,v31
937

938
	vncipherlast	$out0,$out0,$ivec
939
	vncipherlast	$out1,$out1,$in0
940
	 lvx_u		$in0,$x00,$inp		# load next input block
941
	vncipherlast	$out2,$out2,$in1
942
	 lvx_u		$in1,$x10,$inp
943
	vncipherlast	$out3,$out3,$in2
944
	 le?vperm	$in0,$in0,$in0,$inpperm
945
	 lvx_u		$in2,$x20,$inp
946
	vncipherlast	$out4,$out4,$in3
947
	 le?vperm	$in1,$in1,$in1,$inpperm
948
	 lvx_u		$in3,$x30,$inp
949
	vncipherlast	$out5,$out5,$in4
950
	 le?vperm	$in2,$in2,$in2,$inpperm
951
	 lvx_u		$in4,$x40,$inp
952
	vncipherlast	$out6,$out6,$in5
953
	 le?vperm	$in3,$in3,$in3,$inpperm
954
	 lvx_u		$in5,$x50,$inp
955
	vncipherlast	$out7,$out7,$in6
956
	 le?vperm	$in4,$in4,$in4,$inpperm
957
	 lvx_u		$in6,$x60,$inp
958
	vmr		$ivec,$in7
959
	 le?vperm	$in5,$in5,$in5,$inpperm
960
	 lvx_u		$in7,$x70,$inp
961
	 addi		$inp,$inp,0x80
962

963
	le?vperm	$out0,$out0,$out0,$inpperm
964
	le?vperm	$out1,$out1,$out1,$inpperm
965
	stvx_u		$out0,$x00,$out
966
	 le?vperm	$in6,$in6,$in6,$inpperm
967
	 vxor		$out0,$in0,$rndkey0
968
	le?vperm	$out2,$out2,$out2,$inpperm
969
	stvx_u		$out1,$x10,$out
970
	 le?vperm	$in7,$in7,$in7,$inpperm
971
	 vxor		$out1,$in1,$rndkey0
972
	le?vperm	$out3,$out3,$out3,$inpperm
973
	stvx_u		$out2,$x20,$out
974
	 vxor		$out2,$in2,$rndkey0
975
	le?vperm	$out4,$out4,$out4,$inpperm
976
	stvx_u		$out3,$x30,$out
977
	 vxor		$out3,$in3,$rndkey0
978
	le?vperm	$out5,$out5,$out5,$inpperm
979
	stvx_u		$out4,$x40,$out
980
	 vxor		$out4,$in4,$rndkey0
981
	le?vperm	$out6,$out6,$out6,$inpperm
982
	stvx_u		$out5,$x50,$out
983
	 vxor		$out5,$in5,$rndkey0
984
	le?vperm	$out7,$out7,$out7,$inpperm
985
	stvx_u		$out6,$x60,$out
986
	 vxor		$out6,$in6,$rndkey0
987
	stvx_u		$out7,$x70,$out
988
	addi		$out,$out,0x80
989
	 vxor		$out7,$in7,$rndkey0
990

991
	mtctr		$rounds
992
	beq		Loop_cbc_dec8x		# did $len-=128 borrow?
993

994
	addic.		$len,$len,128
995
	beq		Lcbc_dec8x_done
996
	nop
997
	nop
998

999
Loop_cbc_dec8x_tail:				# up to 7 "words" tail...
1000
	vncipher	$out1,$out1,v24
1001
	vncipher	$out2,$out2,v24
1002
	vncipher	$out3,$out3,v24
1003
	vncipher	$out4,$out4,v24
1004
	vncipher	$out5,$out5,v24
1005
	vncipher	$out6,$out6,v24
1006
	vncipher	$out7,$out7,v24
1007
	lvx		v24,$x20,$key_		# round[3]
1008
	addi		$key_,$key_,0x20
1009

1010
	vncipher	$out1,$out1,v25
1011
	vncipher	$out2,$out2,v25
1012
	vncipher	$out3,$out3,v25
1013
	vncipher	$out4,$out4,v25
1014
	vncipher	$out5,$out5,v25
1015
	vncipher	$out6,$out6,v25
1016
	vncipher	$out7,$out7,v25
1017
	lvx		v25,$x10,$key_		# round[4]
1018
	bdnz		Loop_cbc_dec8x_tail
1019

1020
	vncipher	$out1,$out1,v24
1021
	vncipher	$out2,$out2,v24
1022
	vncipher	$out3,$out3,v24
1023
	vncipher	$out4,$out4,v24
1024
	vncipher	$out5,$out5,v24
1025
	vncipher	$out6,$out6,v24
1026
	vncipher	$out7,$out7,v24
1027

1028
	vncipher	$out1,$out1,v25
1029
	vncipher	$out2,$out2,v25
1030
	vncipher	$out3,$out3,v25
1031
	vncipher	$out4,$out4,v25
1032
	vncipher	$out5,$out5,v25
1033
	vncipher	$out6,$out6,v25
1034
	vncipher	$out7,$out7,v25
1035

1036
	vncipher	$out1,$out1,v26
1037
	vncipher	$out2,$out2,v26
1038
	vncipher	$out3,$out3,v26
1039
	vncipher	$out4,$out4,v26
1040
	vncipher	$out5,$out5,v26
1041
	vncipher	$out6,$out6,v26
1042
	vncipher	$out7,$out7,v26
1043

1044
	vncipher	$out1,$out1,v27
1045
	vncipher	$out2,$out2,v27
1046
	vncipher	$out3,$out3,v27
1047
	vncipher	$out4,$out4,v27
1048
	vncipher	$out5,$out5,v27
1049
	vncipher	$out6,$out6,v27
1050
	vncipher	$out7,$out7,v27
1051

1052
	vncipher	$out1,$out1,v28
1053
	vncipher	$out2,$out2,v28
1054
	vncipher	$out3,$out3,v28
1055
	vncipher	$out4,$out4,v28
1056
	vncipher	$out5,$out5,v28
1057
	vncipher	$out6,$out6,v28
1058
	vncipher	$out7,$out7,v28
1059

1060
	vncipher	$out1,$out1,v29
1061
	vncipher	$out2,$out2,v29
1062
	vncipher	$out3,$out3,v29
1063
	vncipher	$out4,$out4,v29
1064
	vncipher	$out5,$out5,v29
1065
	vncipher	$out6,$out6,v29
1066
	vncipher	$out7,$out7,v29
1067

1068
	vncipher	$out1,$out1,v30
1069
	 vxor		$ivec,$ivec,v31		# last round key
1070
	vncipher	$out2,$out2,v30
1071
	 vxor		$in1,$in1,v31
1072
	vncipher	$out3,$out3,v30
1073
	 vxor		$in2,$in2,v31
1074
	vncipher	$out4,$out4,v30
1075
	 vxor		$in3,$in3,v31
1076
	vncipher	$out5,$out5,v30
1077
	 vxor		$in4,$in4,v31
1078
	vncipher	$out6,$out6,v30
1079
	 vxor		$in5,$in5,v31
1080
	vncipher	$out7,$out7,v30
1081
	 vxor		$in6,$in6,v31
1082

1083
	cmplwi		$len,32			# switch($len)
1084
	blt		Lcbc_dec8x_one
1085
	nop
1086
	beq		Lcbc_dec8x_two
1087
	cmplwi		$len,64
1088
	blt		Lcbc_dec8x_three
1089
	nop
1090
	beq		Lcbc_dec8x_four
1091
	cmplwi		$len,96
1092
	blt		Lcbc_dec8x_five
1093
	nop
1094
	beq		Lcbc_dec8x_six
1095

1096
Lcbc_dec8x_seven:
1097
	vncipherlast	$out1,$out1,$ivec
1098
	vncipherlast	$out2,$out2,$in1
1099
	vncipherlast	$out3,$out3,$in2
1100
	vncipherlast	$out4,$out4,$in3
1101
	vncipherlast	$out5,$out5,$in4
1102
	vncipherlast	$out6,$out6,$in5
1103
	vncipherlast	$out7,$out7,$in6
1104
	vmr		$ivec,$in7
1105

1106
	le?vperm	$out1,$out1,$out1,$inpperm
1107
	le?vperm	$out2,$out2,$out2,$inpperm
1108
	stvx_u		$out1,$x00,$out
1109
	le?vperm	$out3,$out3,$out3,$inpperm
1110
	stvx_u		$out2,$x10,$out
1111
	le?vperm	$out4,$out4,$out4,$inpperm
1112
	stvx_u		$out3,$x20,$out
1113
	le?vperm	$out5,$out5,$out5,$inpperm
1114
	stvx_u		$out4,$x30,$out
1115
	le?vperm	$out6,$out6,$out6,$inpperm
1116
	stvx_u		$out5,$x40,$out
1117
	le?vperm	$out7,$out7,$out7,$inpperm
1118
	stvx_u		$out6,$x50,$out
1119
	stvx_u		$out7,$x60,$out
1120
	addi		$out,$out,0x70
1121
	b		Lcbc_dec8x_done
1122

1123
.align	5
1124
Lcbc_dec8x_six:
1125
	vncipherlast	$out2,$out2,$ivec
1126
	vncipherlast	$out3,$out3,$in2
1127
	vncipherlast	$out4,$out4,$in3
1128
	vncipherlast	$out5,$out5,$in4
1129
	vncipherlast	$out6,$out6,$in5
1130
	vncipherlast	$out7,$out7,$in6
1131
	vmr		$ivec,$in7
1132

1133
	le?vperm	$out2,$out2,$out2,$inpperm
1134
	le?vperm	$out3,$out3,$out3,$inpperm
1135
	stvx_u		$out2,$x00,$out
1136
	le?vperm	$out4,$out4,$out4,$inpperm
1137
	stvx_u		$out3,$x10,$out
1138
	le?vperm	$out5,$out5,$out5,$inpperm
1139
	stvx_u		$out4,$x20,$out
1140
	le?vperm	$out6,$out6,$out6,$inpperm
1141
	stvx_u		$out5,$x30,$out
1142
	le?vperm	$out7,$out7,$out7,$inpperm
1143
	stvx_u		$out6,$x40,$out
1144
	stvx_u		$out7,$x50,$out
1145
	addi		$out,$out,0x60
1146
	b		Lcbc_dec8x_done
1147

1148
.align	5
1149
Lcbc_dec8x_five:
1150
	vncipherlast	$out3,$out3,$ivec
1151
	vncipherlast	$out4,$out4,$in3
1152
	vncipherlast	$out5,$out5,$in4
1153
	vncipherlast	$out6,$out6,$in5
1154
	vncipherlast	$out7,$out7,$in6
1155
	vmr		$ivec,$in7
1156

1157
	le?vperm	$out3,$out3,$out3,$inpperm
1158
	le?vperm	$out4,$out4,$out4,$inpperm
1159
	stvx_u		$out3,$x00,$out
1160
	le?vperm	$out5,$out5,$out5,$inpperm
1161
	stvx_u		$out4,$x10,$out
1162
	le?vperm	$out6,$out6,$out6,$inpperm
1163
	stvx_u		$out5,$x20,$out
1164
	le?vperm	$out7,$out7,$out7,$inpperm
1165
	stvx_u		$out6,$x30,$out
1166
	stvx_u		$out7,$x40,$out
1167
	addi		$out,$out,0x50
1168
	b		Lcbc_dec8x_done
1169

1170
.align	5
1171
Lcbc_dec8x_four:
1172
	vncipherlast	$out4,$out4,$ivec
1173
	vncipherlast	$out5,$out5,$in4
1174
	vncipherlast	$out6,$out6,$in5
1175
	vncipherlast	$out7,$out7,$in6
1176
	vmr		$ivec,$in7
1177

1178
	le?vperm	$out4,$out4,$out4,$inpperm
1179
	le?vperm	$out5,$out5,$out5,$inpperm
1180
	stvx_u		$out4,$x00,$out
1181
	le?vperm	$out6,$out6,$out6,$inpperm
1182
	stvx_u		$out5,$x10,$out
1183
	le?vperm	$out7,$out7,$out7,$inpperm
1184
	stvx_u		$out6,$x20,$out
1185
	stvx_u		$out7,$x30,$out
1186
	addi		$out,$out,0x40
1187
	b		Lcbc_dec8x_done
1188

1189
.align	5
1190
Lcbc_dec8x_three:
1191
	vncipherlast	$out5,$out5,$ivec
1192
	vncipherlast	$out6,$out6,$in5
1193
	vncipherlast	$out7,$out7,$in6
1194
	vmr		$ivec,$in7
1195

1196
	le?vperm	$out5,$out5,$out5,$inpperm
1197
	le?vperm	$out6,$out6,$out6,$inpperm
1198
	stvx_u		$out5,$x00,$out
1199
	le?vperm	$out7,$out7,$out7,$inpperm
1200
	stvx_u		$out6,$x10,$out
1201
	stvx_u		$out7,$x20,$out
1202
	addi		$out,$out,0x30
1203
	b		Lcbc_dec8x_done
1204

1205
.align	5
1206
Lcbc_dec8x_two:
1207
	vncipherlast	$out6,$out6,$ivec
1208
	vncipherlast	$out7,$out7,$in6
1209
	vmr		$ivec,$in7
1210

1211
	le?vperm	$out6,$out6,$out6,$inpperm
1212
	le?vperm	$out7,$out7,$out7,$inpperm
1213
	stvx_u		$out6,$x00,$out
1214
	stvx_u		$out7,$x10,$out
1215
	addi		$out,$out,0x20
1216
	b		Lcbc_dec8x_done
1217

1218
.align	5
1219
Lcbc_dec8x_one:
1220
	vncipherlast	$out7,$out7,$ivec
1221
	vmr		$ivec,$in7
1222

1223
	le?vperm	$out7,$out7,$out7,$inpperm
1224
	stvx_u		$out7,0,$out
1225
	addi		$out,$out,0x10
1226

1227
Lcbc_dec8x_done:
1228
	le?vperm	$ivec,$ivec,$ivec,$inpperm
1229
	stvx_u		$ivec,0,$ivp		# write [unaligned] iv
1230

1231
	li		r10,`$FRAME+15`
1232
	li		r11,`$FRAME+31`
1233
	stvx		$inpperm,r10,$sp	# wipe copies of round keys
1234
	addi		r10,r10,32
1235
	stvx		$inpperm,r11,$sp
1236
	addi		r11,r11,32
1237
	stvx		$inpperm,r10,$sp
1238
	addi		r10,r10,32
1239
	stvx		$inpperm,r11,$sp
1240
	addi		r11,r11,32
1241
	stvx		$inpperm,r10,$sp
1242
	addi		r10,r10,32
1243
	stvx		$inpperm,r11,$sp
1244
	addi		r11,r11,32
1245
	stvx		$inpperm,r10,$sp
1246
	addi		r10,r10,32
1247
	stvx		$inpperm,r11,$sp
1248
	addi		r11,r11,32
1249

1250
	mtspr		256,$vrsave
1251
	lvx		v20,r10,$sp		# ABI says so
1252
	addi		r10,r10,32
1253
	lvx		v21,r11,$sp
1254
	addi		r11,r11,32
1255
	lvx		v22,r10,$sp
1256
	addi		r10,r10,32
1257
	lvx		v23,r11,$sp
1258
	addi		r11,r11,32
1259
	lvx		v24,r10,$sp
1260
	addi		r10,r10,32
1261
	lvx		v25,r11,$sp
1262
	addi		r11,r11,32
1263
	lvx		v26,r10,$sp
1264
	addi		r10,r10,32
1265
	lvx		v27,r11,$sp
1266
	addi		r11,r11,32
1267
	lvx		v28,r10,$sp
1268
	addi		r10,r10,32
1269
	lvx		v29,r11,$sp
1270
	addi		r11,r11,32
1271
	lvx		v30,r10,$sp
1272
	lvx		v31,r11,$sp
1273
	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1274
	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1275
	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1276
	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1277
	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1278
	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1279
	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1280
	blr
1281
	.long		0
1282
	.byte		0,12,0x14,0,0x80,6,6,0
1283
	.long		0
1284
.size	.${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
1285
___
1286
}}	}}}
1287

1288
#########################################################################
1289
{{{	# CTR procedure[s]						#
1290

1291
####################### WARNING: Here be dragons! #######################
1292
#
1293
# This code is written as 'ctr32', based on a 32-bit counter used
1294
# upstream. The kernel does *not* use a 32-bit counter. The kernel uses
1295
# a 128-bit counter.
1296
#
1297
# This leads to subtle changes from the upstream code: the counter
1298
# is incremented with vaddu_q_m rather than vaddu_w_m. This occurs in
1299
# both the bulk (8 blocks at a time) path, and in the individual block
1300
# path. Be aware of this when doing updates.
1301
#
1302
# See:
1303
# 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug")
1304
# 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword")
1305
# https://github.com/openssl/openssl/pull/8942
1306
#
1307
#########################################################################
1308
my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));
1309
my ($rndkey0,$rndkey1,$inout,$tmp)=		map("v$_",(0..3));
1310
my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=
1311
						map("v$_",(4..11));
1312
my $dat=$tmp;
1313

1314
$code.=<<___;
1315
.globl	.${prefix}_ctr32_encrypt_blocks
1316
	${UCMP}i	$len,1
1317
	bltlr-
1318

1319
	lis		r0,0xfff0
1320
	mfspr		$vrsave,256
1321
	mtspr		256,r0
1322

1323
	li		$idx,15
1324
	vxor		$rndkey0,$rndkey0,$rndkey0
1325
	le?vspltisb	$tmp,0x0f
1326

1327
	lvx		$ivec,0,$ivp		# load [unaligned] iv
1328
	lvsl		$inpperm,0,$ivp
1329
	lvx		$inptail,$idx,$ivp
1330
	 vspltisb	$one,1
1331
	le?vxor		$inpperm,$inpperm,$tmp
1332
	vperm		$ivec,$ivec,$inptail,$inpperm
1333
	 vsldoi		$one,$rndkey0,$one,1
1334

1335
	neg		r11,$inp
1336
	?lvsl		$keyperm,0,$key		# prepare for unaligned key
1337
	lwz		$rounds,240($key)
1338

1339
	lvsr		$inpperm,0,r11		# prepare for unaligned load
1340
	lvx		$inptail,0,$inp
1341
	addi		$inp,$inp,15		# 15 is not typo
1342
	le?vxor		$inpperm,$inpperm,$tmp
1343

1344
	srwi		$rounds,$rounds,1
1345
	li		$idx,16
1346
	subi		$rounds,$rounds,1
1347

1348
	${UCMP}i	$len,8
1349
	bge		_aesp8_ctr32_encrypt8x
1350

1351
	?lvsr		$outperm,0,$out		# prepare for unaligned store
1352
	vspltisb	$outmask,-1
1353
	lvx		$outhead,0,$out
1354
	?vperm		$outmask,$rndkey0,$outmask,$outperm
1355
	le?vxor		$outperm,$outperm,$tmp
1356

1357
	lvx		$rndkey0,0,$key
1358
	mtctr		$rounds
1359
	lvx		$rndkey1,$idx,$key
1360
	addi		$idx,$idx,16
1361
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
1362
	vxor		$inout,$ivec,$rndkey0
1363
	lvx		$rndkey0,$idx,$key
1364
	addi		$idx,$idx,16
1365
	b		Loop_ctr32_enc
1366

1367
.align	5
1368
Loop_ctr32_enc:
1369
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
1370
	vcipher		$inout,$inout,$rndkey1
1371
	lvx		$rndkey1,$idx,$key
1372
	addi		$idx,$idx,16
1373
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
1374
	vcipher		$inout,$inout,$rndkey0
1375
	lvx		$rndkey0,$idx,$key
1376
	addi		$idx,$idx,16
1377
	bdnz		Loop_ctr32_enc
1378

1379
	vadduqm		$ivec,$ivec,$one	# Kernel change for 128-bit
1380
	 vmr		$dat,$inptail
1381
	 lvx		$inptail,0,$inp
1382
	 addi		$inp,$inp,16
1383
	 subic.		$len,$len,1		# blocks--
1384

1385
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
1386
	vcipher		$inout,$inout,$rndkey1
1387
	lvx		$rndkey1,$idx,$key
1388
	 vperm		$dat,$dat,$inptail,$inpperm
1389
	 li		$idx,16
1390
	?vperm		$rndkey1,$rndkey0,$rndkey1,$keyperm
1391
	 lvx		$rndkey0,0,$key
1392
	vxor		$dat,$dat,$rndkey1	# last round key
1393
	vcipherlast	$inout,$inout,$dat
1394

1395
	 lvx		$rndkey1,$idx,$key
1396
	 addi		$idx,$idx,16
1397
	vperm		$inout,$inout,$inout,$outperm
1398
	vsel		$dat,$outhead,$inout,$outmask
1399
	 mtctr		$rounds
1400
	 ?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
1401
	vmr		$outhead,$inout
1402
	 vxor		$inout,$ivec,$rndkey0
1403
	 lvx		$rndkey0,$idx,$key
1404
	 addi		$idx,$idx,16
1405
	stvx		$dat,0,$out
1406
	addi		$out,$out,16
1407
	bne		Loop_ctr32_enc
1408

1409
	addi		$out,$out,-1
1410
	lvx		$inout,0,$out		# redundant in aligned case
1411
	vsel		$inout,$outhead,$inout,$outmask
1412
	stvx		$inout,0,$out
1413

1414
	mtspr		256,$vrsave
1415
	blr
1416
	.long		0
1417
	.byte		0,12,0x14,0,0,0,6,0
1418
	.long		0
1419
___
1420
#########################################################################
1421
{{	# Optimized CTR procedure					#
1422
my $key_="r11";
1423
my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
1424
my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));
1425
my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22));
1426
my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
1427
			# v26-v31 last 6 round keys
1428
my ($tmp,$keyperm)=($in3,$in4);	# aliases with "caller", redundant assignment
1429
my ($two,$three,$four)=($outhead,$outperm,$outmask);
1430

1431
$code.=<<___;
1432
.align	5
1433
_aesp8_ctr32_encrypt8x:
1434
	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
1435
	li		r10,`$FRAME+8*16+15`
1436
	li		r11,`$FRAME+8*16+31`
1437
	stvx		v20,r10,$sp		# ABI says so
1438
	addi		r10,r10,32
1439
	stvx		v21,r11,$sp
1440
	addi		r11,r11,32
1441
	stvx		v22,r10,$sp
1442
	addi		r10,r10,32
1443
	stvx		v23,r11,$sp
1444
	addi		r11,r11,32
1445
	stvx		v24,r10,$sp
1446
	addi		r10,r10,32
1447
	stvx		v25,r11,$sp
1448
	addi		r11,r11,32
1449
	stvx		v26,r10,$sp
1450
	addi		r10,r10,32
1451
	stvx		v27,r11,$sp
1452
	addi		r11,r11,32
1453
	stvx		v28,r10,$sp
1454
	addi		r10,r10,32
1455
	stvx		v29,r11,$sp
1456
	addi		r11,r11,32
1457
	stvx		v30,r10,$sp
1458
	stvx		v31,r11,$sp
1459
	li		r0,-1
1460
	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
1461
	li		$x10,0x10
1462
	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1463
	li		$x20,0x20
1464
	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1465
	li		$x30,0x30
1466
	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1467
	li		$x40,0x40
1468
	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1469
	li		$x50,0x50
1470
	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1471
	li		$x60,0x60
1472
	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1473
	li		$x70,0x70
1474
	mtspr		256,r0
1475

1476
	subi		$rounds,$rounds,3	# -4 in total
1477

1478
	lvx		$rndkey0,$x00,$key	# load key schedule
1479
	lvx		v30,$x10,$key
1480
	addi		$key,$key,0x20
1481
	lvx		v31,$x00,$key
1482
	?vperm		$rndkey0,$rndkey0,v30,$keyperm
1483
	addi		$key_,$sp,$FRAME+15
1484
	mtctr		$rounds
1485

1486
Load_ctr32_enc_key:
1487
	?vperm		v24,v30,v31,$keyperm
1488
	lvx		v30,$x10,$key
1489
	addi		$key,$key,0x20
1490
	stvx		v24,$x00,$key_		# off-load round[1]
1491
	?vperm		v25,v31,v30,$keyperm
1492
	lvx		v31,$x00,$key
1493
	stvx		v25,$x10,$key_		# off-load round[2]
1494
	addi		$key_,$key_,0x20
1495
	bdnz		Load_ctr32_enc_key
1496

1497
	lvx		v26,$x10,$key
1498
	?vperm		v24,v30,v31,$keyperm
1499
	lvx		v27,$x20,$key
1500
	stvx		v24,$x00,$key_		# off-load round[3]
1501
	?vperm		v25,v31,v26,$keyperm
1502
	lvx		v28,$x30,$key
1503
	stvx		v25,$x10,$key_		# off-load round[4]
1504
	addi		$key_,$sp,$FRAME+15	# rewind $key_
1505
	?vperm		v26,v26,v27,$keyperm
1506
	lvx		v29,$x40,$key
1507
	?vperm		v27,v27,v28,$keyperm
1508
	lvx		v30,$x50,$key
1509
	?vperm		v28,v28,v29,$keyperm
1510
	lvx		v31,$x60,$key
1511
	?vperm		v29,v29,v30,$keyperm
1512
	lvx		$out0,$x70,$key		# borrow $out0
1513
	?vperm		v30,v30,v31,$keyperm
1514
	lvx		v24,$x00,$key_		# pre-load round[1]
1515
	?vperm		v31,v31,$out0,$keyperm
1516
	lvx		v25,$x10,$key_		# pre-load round[2]
1517

1518
	vadduqm		$two,$one,$one
1519
	subi		$inp,$inp,15		# undo "caller"
1520
	$SHL		$len,$len,4
1521

1522
	vadduqm		$out1,$ivec,$one	# counter values ...
1523
	vadduqm		$out2,$ivec,$two	# (do all ctr adds as 128-bit)
1524
	vxor		$out0,$ivec,$rndkey0	# ... xored with rndkey[0]
1525
	 le?li		$idx,8
1526
	vadduqm		$out3,$out1,$two
1527
	vxor		$out1,$out1,$rndkey0
1528
	 le?lvsl	$inpperm,0,$idx
1529
	vadduqm		$out4,$out2,$two
1530
	vxor		$out2,$out2,$rndkey0
1531
	 le?vspltisb	$tmp,0x0f
1532
	vadduqm		$out5,$out3,$two
1533
	vxor		$out3,$out3,$rndkey0
1534
	 le?vxor	$inpperm,$inpperm,$tmp	# transform for lvx_u/stvx_u
1535
	vadduqm		$out6,$out4,$two
1536
	vxor		$out4,$out4,$rndkey0
1537
	vadduqm		$out7,$out5,$two
1538
	vxor		$out5,$out5,$rndkey0
1539
	vadduqm		$ivec,$out6,$two	# next counter value
1540
	vxor		$out6,$out6,$rndkey0
1541
	vxor		$out7,$out7,$rndkey0
1542

1543
	mtctr		$rounds
1544
	b		Loop_ctr32_enc8x
1545
.align	5
1546
Loop_ctr32_enc8x:
1547
	vcipher 	$out0,$out0,v24
1548
	vcipher 	$out1,$out1,v24
1549
	vcipher 	$out2,$out2,v24
1550
	vcipher 	$out3,$out3,v24
1551
	vcipher 	$out4,$out4,v24
1552
	vcipher 	$out5,$out5,v24
1553
	vcipher 	$out6,$out6,v24
1554
	vcipher 	$out7,$out7,v24
1555
Loop_ctr32_enc8x_middle:
1556
	lvx		v24,$x20,$key_		# round[3]
1557
	addi		$key_,$key_,0x20
1558

1559
	vcipher 	$out0,$out0,v25
1560
	vcipher 	$out1,$out1,v25
1561
	vcipher 	$out2,$out2,v25
1562
	vcipher 	$out3,$out3,v25
1563
	vcipher 	$out4,$out4,v25
1564
	vcipher 	$out5,$out5,v25
1565
	vcipher 	$out6,$out6,v25
1566
	vcipher 	$out7,$out7,v25
1567
	lvx		v25,$x10,$key_		# round[4]
1568
	bdnz		Loop_ctr32_enc8x
1569

1570
	subic		r11,$len,256		# $len-256, borrow $key_
1571
	vcipher 	$out0,$out0,v24
1572
	vcipher 	$out1,$out1,v24
1573
	vcipher 	$out2,$out2,v24
1574
	vcipher 	$out3,$out3,v24
1575
	vcipher 	$out4,$out4,v24
1576
	vcipher 	$out5,$out5,v24
1577
	vcipher 	$out6,$out6,v24
1578
	vcipher 	$out7,$out7,v24
1579

1580
	subfe		r0,r0,r0		# borrow?-1:0
1581
	vcipher 	$out0,$out0,v25
1582
	vcipher 	$out1,$out1,v25
1583
	vcipher 	$out2,$out2,v25
1584
	vcipher 	$out3,$out3,v25
1585
	vcipher 	$out4,$out4,v25
1586
	vcipher		$out5,$out5,v25
1587
	vcipher		$out6,$out6,v25
1588
	vcipher		$out7,$out7,v25
1589

1590
	and		r0,r0,r11
1591
	addi		$key_,$sp,$FRAME+15	# rewind $key_
1592
	vcipher		$out0,$out0,v26
1593
	vcipher		$out1,$out1,v26
1594
	vcipher		$out2,$out2,v26
1595
	vcipher		$out3,$out3,v26
1596
	vcipher		$out4,$out4,v26
1597
	vcipher		$out5,$out5,v26
1598
	vcipher		$out6,$out6,v26
1599
	vcipher		$out7,$out7,v26
1600
	lvx		v24,$x00,$key_		# re-pre-load round[1]
1601

1602
	subic		$len,$len,129		# $len-=129
1603
	vcipher		$out0,$out0,v27
1604
	addi		$len,$len,1		# $len-=128 really
1605
	vcipher		$out1,$out1,v27
1606
	vcipher		$out2,$out2,v27
1607
	vcipher		$out3,$out3,v27
1608
	vcipher		$out4,$out4,v27
1609
	vcipher		$out5,$out5,v27
1610
	vcipher		$out6,$out6,v27
1611
	vcipher		$out7,$out7,v27
1612
	lvx		v25,$x10,$key_		# re-pre-load round[2]
1613

1614
	vcipher		$out0,$out0,v28
1615
	 lvx_u		$in0,$x00,$inp		# load input
1616
	vcipher		$out1,$out1,v28
1617
	 lvx_u		$in1,$x10,$inp
1618
	vcipher		$out2,$out2,v28
1619
	 lvx_u		$in2,$x20,$inp
1620
	vcipher		$out3,$out3,v28
1621
	 lvx_u		$in3,$x30,$inp
1622
	vcipher		$out4,$out4,v28
1623
	 lvx_u		$in4,$x40,$inp
1624
	vcipher		$out5,$out5,v28
1625
	 lvx_u		$in5,$x50,$inp
1626
	vcipher		$out6,$out6,v28
1627
	 lvx_u		$in6,$x60,$inp
1628
	vcipher		$out7,$out7,v28
1629
	 lvx_u		$in7,$x70,$inp
1630
	 addi		$inp,$inp,0x80
1631

1632
	vcipher		$out0,$out0,v29
1633
	 le?vperm	$in0,$in0,$in0,$inpperm
1634
	vcipher		$out1,$out1,v29
1635
	 le?vperm	$in1,$in1,$in1,$inpperm
1636
	vcipher		$out2,$out2,v29
1637
	 le?vperm	$in2,$in2,$in2,$inpperm
1638
	vcipher		$out3,$out3,v29
1639
	 le?vperm	$in3,$in3,$in3,$inpperm
1640
	vcipher		$out4,$out4,v29
1641
	 le?vperm	$in4,$in4,$in4,$inpperm
1642
	vcipher		$out5,$out5,v29
1643
	 le?vperm	$in5,$in5,$in5,$inpperm
1644
	vcipher		$out6,$out6,v29
1645
	 le?vperm	$in6,$in6,$in6,$inpperm
1646
	vcipher		$out7,$out7,v29
1647
	 le?vperm	$in7,$in7,$in7,$inpperm
1648

1649
	add		$inp,$inp,r0		# $inp is adjusted in such
1650
						# way that at exit from the
1651
						# loop inX-in7 are loaded
1652
						# with last "words"
1653
	subfe.		r0,r0,r0		# borrow?-1:0
1654
	vcipher		$out0,$out0,v30
1655
	 vxor		$in0,$in0,v31		# xor with last round key
1656
	vcipher		$out1,$out1,v30
1657
	 vxor		$in1,$in1,v31
1658
	vcipher		$out2,$out2,v30
1659
	 vxor		$in2,$in2,v31
1660
	vcipher		$out3,$out3,v30
1661
	 vxor		$in3,$in3,v31
1662
	vcipher		$out4,$out4,v30
1663
	 vxor		$in4,$in4,v31
1664
	vcipher		$out5,$out5,v30
1665
	 vxor		$in5,$in5,v31
1666
	vcipher		$out6,$out6,v30
1667
	 vxor		$in6,$in6,v31
1668
	vcipher		$out7,$out7,v30
1669
	 vxor		$in7,$in7,v31
1670

1671
	bne		Lctr32_enc8x_break	# did $len-129 borrow?
1672

1673
	vcipherlast	$in0,$out0,$in0
1674
	vcipherlast	$in1,$out1,$in1
1675
	 vadduqm	$out1,$ivec,$one	# counter values ...
1676
	vcipherlast	$in2,$out2,$in2
1677
	 vadduqm	$out2,$ivec,$two
1678
	 vxor		$out0,$ivec,$rndkey0	# ... xored with rndkey[0]
1679
	vcipherlast	$in3,$out3,$in3
1680
	 vadduqm	$out3,$out1,$two
1681
	 vxor		$out1,$out1,$rndkey0
1682
	vcipherlast	$in4,$out4,$in4
1683
	 vadduqm	$out4,$out2,$two
1684
	 vxor		$out2,$out2,$rndkey0
1685
	vcipherlast	$in5,$out5,$in5
1686
	 vadduqm	$out5,$out3,$two
1687
	 vxor		$out3,$out3,$rndkey0
1688
	vcipherlast	$in6,$out6,$in6
1689
	 vadduqm	$out6,$out4,$two
1690
	 vxor		$out4,$out4,$rndkey0
1691
	vcipherlast	$in7,$out7,$in7
1692
	 vadduqm	$out7,$out5,$two
1693
	 vxor		$out5,$out5,$rndkey0
1694
	le?vperm	$in0,$in0,$in0,$inpperm
1695
	 vadduqm	$ivec,$out6,$two	# next counter value
1696
	 vxor		$out6,$out6,$rndkey0
1697
	le?vperm	$in1,$in1,$in1,$inpperm
1698
	 vxor		$out7,$out7,$rndkey0
1699
	mtctr		$rounds
1700

1701
	 vcipher	$out0,$out0,v24
1702
	stvx_u		$in0,$x00,$out
1703
	le?vperm	$in2,$in2,$in2,$inpperm
1704
	 vcipher	$out1,$out1,v24
1705
	stvx_u		$in1,$x10,$out
1706
	le?vperm	$in3,$in3,$in3,$inpperm
1707
	 vcipher	$out2,$out2,v24
1708
	stvx_u		$in2,$x20,$out
1709
	le?vperm	$in4,$in4,$in4,$inpperm
1710
	 vcipher	$out3,$out3,v24
1711
	stvx_u		$in3,$x30,$out
1712
	le?vperm	$in5,$in5,$in5,$inpperm
1713
	 vcipher	$out4,$out4,v24
1714
	stvx_u		$in4,$x40,$out
1715
	le?vperm	$in6,$in6,$in6,$inpperm
1716
	 vcipher	$out5,$out5,v24
1717
	stvx_u		$in5,$x50,$out
1718
	le?vperm	$in7,$in7,$in7,$inpperm
1719
	 vcipher	$out6,$out6,v24
1720
	stvx_u		$in6,$x60,$out
1721
	 vcipher	$out7,$out7,v24
1722
	stvx_u		$in7,$x70,$out
1723
	addi		$out,$out,0x80
1724

1725
	b		Loop_ctr32_enc8x_middle
1726

1727
.align	5
1728
Lctr32_enc8x_break:
1729
	cmpwi		$len,-0x60
1730
	blt		Lctr32_enc8x_one
1731
	nop
1732
	beq		Lctr32_enc8x_two
1733
	cmpwi		$len,-0x40
1734
	blt		Lctr32_enc8x_three
1735
	nop
1736
	beq		Lctr32_enc8x_four
1737
	cmpwi		$len,-0x20
1738
	blt		Lctr32_enc8x_five
1739
	nop
1740
	beq		Lctr32_enc8x_six
1741
	cmpwi		$len,0x00
1742
	blt		Lctr32_enc8x_seven
1743

1744
Lctr32_enc8x_eight:
1745
	vcipherlast	$out0,$out0,$in0
1746
	vcipherlast	$out1,$out1,$in1
1747
	vcipherlast	$out2,$out2,$in2
1748
	vcipherlast	$out3,$out3,$in3
1749
	vcipherlast	$out4,$out4,$in4
1750
	vcipherlast	$out5,$out5,$in5
1751
	vcipherlast	$out6,$out6,$in6
1752
	vcipherlast	$out7,$out7,$in7
1753

1754
	le?vperm	$out0,$out0,$out0,$inpperm
1755
	le?vperm	$out1,$out1,$out1,$inpperm
1756
	stvx_u		$out0,$x00,$out
1757
	le?vperm	$out2,$out2,$out2,$inpperm
1758
	stvx_u		$out1,$x10,$out
1759
	le?vperm	$out3,$out3,$out3,$inpperm
1760
	stvx_u		$out2,$x20,$out
1761
	le?vperm	$out4,$out4,$out4,$inpperm
1762
	stvx_u		$out3,$x30,$out
1763
	le?vperm	$out5,$out5,$out5,$inpperm
1764
	stvx_u		$out4,$x40,$out
1765
	le?vperm	$out6,$out6,$out6,$inpperm
1766
	stvx_u		$out5,$x50,$out
1767
	le?vperm	$out7,$out7,$out7,$inpperm
1768
	stvx_u		$out6,$x60,$out
1769
	stvx_u		$out7,$x70,$out
1770
	addi		$out,$out,0x80
1771
	b		Lctr32_enc8x_done
1772

1773
.align	5
1774
Lctr32_enc8x_seven:
1775
	vcipherlast	$out0,$out0,$in1
1776
	vcipherlast	$out1,$out1,$in2
1777
	vcipherlast	$out2,$out2,$in3
1778
	vcipherlast	$out3,$out3,$in4
1779
	vcipherlast	$out4,$out4,$in5
1780
	vcipherlast	$out5,$out5,$in6
1781
	vcipherlast	$out6,$out6,$in7
1782

1783
	le?vperm	$out0,$out0,$out0,$inpperm
1784
	le?vperm	$out1,$out1,$out1,$inpperm
1785
	stvx_u		$out0,$x00,$out
1786
	le?vperm	$out2,$out2,$out2,$inpperm
1787
	stvx_u		$out1,$x10,$out
1788
	le?vperm	$out3,$out3,$out3,$inpperm
1789
	stvx_u		$out2,$x20,$out
1790
	le?vperm	$out4,$out4,$out4,$inpperm
1791
	stvx_u		$out3,$x30,$out
1792
	le?vperm	$out5,$out5,$out5,$inpperm
1793
	stvx_u		$out4,$x40,$out
1794
	le?vperm	$out6,$out6,$out6,$inpperm
1795
	stvx_u		$out5,$x50,$out
1796
	stvx_u		$out6,$x60,$out
1797
	addi		$out,$out,0x70
1798
	b		Lctr32_enc8x_done
1799

1800
.align	5
1801
Lctr32_enc8x_six:
1802
	vcipherlast	$out0,$out0,$in2
1803
	vcipherlast	$out1,$out1,$in3
1804
	vcipherlast	$out2,$out2,$in4
1805
	vcipherlast	$out3,$out3,$in5
1806
	vcipherlast	$out4,$out4,$in6
1807
	vcipherlast	$out5,$out5,$in7
1808

1809
	le?vperm	$out0,$out0,$out0,$inpperm
1810
	le?vperm	$out1,$out1,$out1,$inpperm
1811
	stvx_u		$out0,$x00,$out
1812
	le?vperm	$out2,$out2,$out2,$inpperm
1813
	stvx_u		$out1,$x10,$out
1814
	le?vperm	$out3,$out3,$out3,$inpperm
1815
	stvx_u		$out2,$x20,$out
1816
	le?vperm	$out4,$out4,$out4,$inpperm
1817
	stvx_u		$out3,$x30,$out
1818
	le?vperm	$out5,$out5,$out5,$inpperm
1819
	stvx_u		$out4,$x40,$out
1820
	stvx_u		$out5,$x50,$out
1821
	addi		$out,$out,0x60
1822
	b		Lctr32_enc8x_done
1823

1824
.align	5
1825
Lctr32_enc8x_five:
1826
	vcipherlast	$out0,$out0,$in3
1827
	vcipherlast	$out1,$out1,$in4
1828
	vcipherlast	$out2,$out2,$in5
1829
	vcipherlast	$out3,$out3,$in6
1830
	vcipherlast	$out4,$out4,$in7
1831

1832
	le?vperm	$out0,$out0,$out0,$inpperm
1833
	le?vperm	$out1,$out1,$out1,$inpperm
1834
	stvx_u		$out0,$x00,$out
1835
	le?vperm	$out2,$out2,$out2,$inpperm
1836
	stvx_u		$out1,$x10,$out
1837
	le?vperm	$out3,$out3,$out3,$inpperm
1838
	stvx_u		$out2,$x20,$out
1839
	le?vperm	$out4,$out4,$out4,$inpperm
1840
	stvx_u		$out3,$x30,$out
1841
	stvx_u		$out4,$x40,$out
1842
	addi		$out,$out,0x50
1843
	b		Lctr32_enc8x_done
1844

1845
.align	5
1846
Lctr32_enc8x_four:
1847
	vcipherlast	$out0,$out0,$in4
1848
	vcipherlast	$out1,$out1,$in5
1849
	vcipherlast	$out2,$out2,$in6
1850
	vcipherlast	$out3,$out3,$in7
1851

1852
	le?vperm	$out0,$out0,$out0,$inpperm
1853
	le?vperm	$out1,$out1,$out1,$inpperm
1854
	stvx_u		$out0,$x00,$out
1855
	le?vperm	$out2,$out2,$out2,$inpperm
1856
	stvx_u		$out1,$x10,$out
1857
	le?vperm	$out3,$out3,$out3,$inpperm
1858
	stvx_u		$out2,$x20,$out
1859
	stvx_u		$out3,$x30,$out
1860
	addi		$out,$out,0x40
1861
	b		Lctr32_enc8x_done
1862

1863
.align	5
1864
Lctr32_enc8x_three:
1865
	vcipherlast	$out0,$out0,$in5
1866
	vcipherlast	$out1,$out1,$in6
1867
	vcipherlast	$out2,$out2,$in7
1868

1869
	le?vperm	$out0,$out0,$out0,$inpperm
1870
	le?vperm	$out1,$out1,$out1,$inpperm
1871
	stvx_u		$out0,$x00,$out
1872
	le?vperm	$out2,$out2,$out2,$inpperm
1873
	stvx_u		$out1,$x10,$out
1874
	stvx_u		$out2,$x20,$out
1875
	addi		$out,$out,0x30
1876
	b		Lctr32_enc8x_done
1877

1878
.align	5
1879
Lctr32_enc8x_two:
1880
	vcipherlast	$out0,$out0,$in6
1881
	vcipherlast	$out1,$out1,$in7
1882

1883
	le?vperm	$out0,$out0,$out0,$inpperm
1884
	le?vperm	$out1,$out1,$out1,$inpperm
1885
	stvx_u		$out0,$x00,$out
1886
	stvx_u		$out1,$x10,$out
1887
	addi		$out,$out,0x20
1888
	b		Lctr32_enc8x_done
1889

1890
.align	5
1891
Lctr32_enc8x_one:
1892
	vcipherlast	$out0,$out0,$in7
1893

1894
	le?vperm	$out0,$out0,$out0,$inpperm
1895
	stvx_u		$out0,0,$out
1896
	addi		$out,$out,0x10
1897

1898
Lctr32_enc8x_done:
1899
	li		r10,`$FRAME+15`
1900
	li		r11,`$FRAME+31`
1901
	stvx		$inpperm,r10,$sp	# wipe copies of round keys
1902
	addi		r10,r10,32
1903
	stvx		$inpperm,r11,$sp
1904
	addi		r11,r11,32
1905
	stvx		$inpperm,r10,$sp
1906
	addi		r10,r10,32
1907
	stvx		$inpperm,r11,$sp
1908
	addi		r11,r11,32
1909
	stvx		$inpperm,r10,$sp
1910
	addi		r10,r10,32
1911
	stvx		$inpperm,r11,$sp
1912
	addi		r11,r11,32
1913
	stvx		$inpperm,r10,$sp
1914
	addi		r10,r10,32
1915
	stvx		$inpperm,r11,$sp
1916
	addi		r11,r11,32
1917

1918
	mtspr		256,$vrsave
1919
	lvx		v20,r10,$sp		# ABI says so
1920
	addi		r10,r10,32
1921
	lvx		v21,r11,$sp
1922
	addi		r11,r11,32
1923
	lvx		v22,r10,$sp
1924
	addi		r10,r10,32
1925
	lvx		v23,r11,$sp
1926
	addi		r11,r11,32
1927
	lvx		v24,r10,$sp
1928
	addi		r10,r10,32
1929
	lvx		v25,r11,$sp
1930
	addi		r11,r11,32
1931
	lvx		v26,r10,$sp
1932
	addi		r10,r10,32
1933
	lvx		v27,r11,$sp
1934
	addi		r11,r11,32
1935
	lvx		v28,r10,$sp
1936
	addi		r10,r10,32
1937
	lvx		v29,r11,$sp
1938
	addi		r11,r11,32
1939
	lvx		v30,r10,$sp
1940
	lvx		v31,r11,$sp
1941
	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1942
	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1943
	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1944
	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1945
	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1946
	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1947
	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1948
	blr
1949
	.long		0
1950
	.byte		0,12,0x14,0,0x80,6,6,0
1951
	.long		0
1952
.size	.${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
1953
___
1954
}}	}}}
1955

1956
#########################################################################
1957
{{{	# XTS procedures						#
1958
# int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len,	#
1959
#                             const AES_KEY *key1, const AES_KEY *key2,	#
1960
#                             [const] unsigned char iv[16]);		#
1961
# If $key2 is NULL, then a "tweak chaining" mode is engaged, in which	#
1962
# input tweak value is assumed to be encrypted already, and last tweak	#
1963
# value, one suitable for consecutive call on same chunk of data, is	#
1964
# written back to original buffer. In addition, in "tweak chaining"	#
1965
# mode only complete input blocks are processed.			#
1966

1967
my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) =	map("r$_",(3..10));
1968
my ($rndkey0,$rndkey1,$inout) =				map("v$_",(0..2));
1969
my ($output,$inptail,$inpperm,$leperm,$keyperm) =	map("v$_",(3..7));
1970
my ($tweak,$seven,$eighty7,$tmp,$tweak1) =		map("v$_",(8..12));
1971
my $taillen = $key2;
1972

1973
   ($inp,$idx) = ($idx,$inp);				# reassign
1974

1975
$code.=<<___;
1976
.globl	.${prefix}_xts_encrypt
1977
	mr		$inp,r3				# reassign
1978
	li		r3,-1
1979
	${UCMP}i	$len,16
1980
	bltlr-
1981

1982
	lis		r0,0xfff0
1983
	mfspr		r12,256				# save vrsave
1984
	li		r11,0
1985
	mtspr		256,r0
1986

1987
	vspltisb	$seven,0x07			# 0x070707..07
1988
	le?lvsl		$leperm,r11,r11
1989
	le?vspltisb	$tmp,0x0f
1990
	le?vxor		$leperm,$leperm,$seven
1991

1992
	li		$idx,15
1993
	lvx		$tweak,0,$ivp			# load [unaligned] iv
1994
	lvsl		$inpperm,0,$ivp
1995
	lvx		$inptail,$idx,$ivp
1996
	le?vxor		$inpperm,$inpperm,$tmp
1997
	vperm		$tweak,$tweak,$inptail,$inpperm
1998

1999
	neg		r11,$inp
2000
	lvsr		$inpperm,0,r11			# prepare for unaligned load
2001
	lvx		$inout,0,$inp
2002
	addi		$inp,$inp,15			# 15 is not typo
2003
	le?vxor		$inpperm,$inpperm,$tmp
2004

2005
	${UCMP}i	$key2,0				# key2==NULL?
2006
	beq		Lxts_enc_no_key2
2007

2008
	?lvsl		$keyperm,0,$key2		# prepare for unaligned key
2009
	lwz		$rounds,240($key2)
2010
	srwi		$rounds,$rounds,1
2011
	subi		$rounds,$rounds,1
2012
	li		$idx,16
2013

2014
	lvx		$rndkey0,0,$key2
2015
	lvx		$rndkey1,$idx,$key2
2016
	addi		$idx,$idx,16
2017
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2018
	vxor		$tweak,$tweak,$rndkey0
2019
	lvx		$rndkey0,$idx,$key2
2020
	addi		$idx,$idx,16
2021
	mtctr		$rounds
2022

2023
Ltweak_xts_enc:
2024
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
2025
	vcipher		$tweak,$tweak,$rndkey1
2026
	lvx		$rndkey1,$idx,$key2
2027
	addi		$idx,$idx,16
2028
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2029
	vcipher		$tweak,$tweak,$rndkey0
2030
	lvx		$rndkey0,$idx,$key2
2031
	addi		$idx,$idx,16
2032
	bdnz		Ltweak_xts_enc
2033

2034
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
2035
	vcipher		$tweak,$tweak,$rndkey1
2036
	lvx		$rndkey1,$idx,$key2
2037
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2038
	vcipherlast	$tweak,$tweak,$rndkey0
2039

2040
	li		$ivp,0				# don't chain the tweak
2041
	b		Lxts_enc
2042

2043
Lxts_enc_no_key2:
2044
	li		$idx,-16
2045
	and		$len,$len,$idx			# in "tweak chaining"
2046
							# mode only complete
2047
							# blocks are processed
2048
Lxts_enc:
2049
	lvx		$inptail,0,$inp
2050
	addi		$inp,$inp,16
2051

2052
	?lvsl		$keyperm,0,$key1		# prepare for unaligned key
2053
	lwz		$rounds,240($key1)
2054
	srwi		$rounds,$rounds,1
2055
	subi		$rounds,$rounds,1
2056
	li		$idx,16
2057

2058
	vslb		$eighty7,$seven,$seven		# 0x808080..80
2059
	vor		$eighty7,$eighty7,$seven	# 0x878787..87
2060
	vspltisb	$tmp,1				# 0x010101..01
2061
	vsldoi		$eighty7,$eighty7,$tmp,15	# 0x870101..01
2062

2063
	${UCMP}i	$len,96
2064
	bge		_aesp8_xts_encrypt6x
2065

2066
	andi.		$taillen,$len,15
2067
	subic		r0,$len,32
2068
	subi		$taillen,$taillen,16
2069
	subfe		r0,r0,r0
2070
	and		r0,r0,$taillen
2071
	add		$inp,$inp,r0
2072

2073
	lvx		$rndkey0,0,$key1
2074
	lvx		$rndkey1,$idx,$key1
2075
	addi		$idx,$idx,16
2076
	vperm		$inout,$inout,$inptail,$inpperm
2077
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2078
	vxor		$inout,$inout,$tweak
2079
	vxor		$inout,$inout,$rndkey0
2080
	lvx		$rndkey0,$idx,$key1
2081
	addi		$idx,$idx,16
2082
	mtctr		$rounds
2083
	b		Loop_xts_enc
2084

2085
.align	5
2086
Loop_xts_enc:
2087
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
2088
	vcipher		$inout,$inout,$rndkey1
2089
	lvx		$rndkey1,$idx,$key1
2090
	addi		$idx,$idx,16
2091
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2092
	vcipher		$inout,$inout,$rndkey0
2093
	lvx		$rndkey0,$idx,$key1
2094
	addi		$idx,$idx,16
2095
	bdnz		Loop_xts_enc
2096

2097
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
2098
	vcipher		$inout,$inout,$rndkey1
2099
	lvx		$rndkey1,$idx,$key1
2100
	li		$idx,16
2101
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2102
	vxor		$rndkey0,$rndkey0,$tweak
2103
	vcipherlast	$output,$inout,$rndkey0
2104

2105
	le?vperm	$tmp,$output,$output,$leperm
2106
	be?nop
2107
	le?stvx_u	$tmp,0,$out
2108
	be?stvx_u	$output,0,$out
2109
	addi		$out,$out,16
2110

2111
	subic.		$len,$len,16
2112
	beq		Lxts_enc_done
2113

2114
	vmr		$inout,$inptail
2115
	lvx		$inptail,0,$inp
2116
	addi		$inp,$inp,16
2117
	lvx		$rndkey0,0,$key1
2118
	lvx		$rndkey1,$idx,$key1
2119
	addi		$idx,$idx,16
2120

2121
	subic		r0,$len,32
2122
	subfe		r0,r0,r0
2123
	and		r0,r0,$taillen
2124
	add		$inp,$inp,r0
2125

2126
	vsrab		$tmp,$tweak,$seven		# next tweak value
2127
	vaddubm		$tweak,$tweak,$tweak
2128
	vsldoi		$tmp,$tmp,$tmp,15
2129
	vand		$tmp,$tmp,$eighty7
2130
	vxor		$tweak,$tweak,$tmp
2131

2132
	vperm		$inout,$inout,$inptail,$inpperm
2133
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2134
	vxor		$inout,$inout,$tweak
2135
	vxor		$output,$output,$rndkey0	# just in case $len<16
2136
	vxor		$inout,$inout,$rndkey0
2137
	lvx		$rndkey0,$idx,$key1
2138
	addi		$idx,$idx,16
2139

2140
	mtctr		$rounds
2141
	${UCMP}i	$len,16
2142
	bge		Loop_xts_enc
2143

2144
	vxor		$output,$output,$tweak
2145
	lvsr		$inpperm,0,$len			# $inpperm is no longer needed
2146
	vxor		$inptail,$inptail,$inptail	# $inptail is no longer needed
2147
	vspltisb	$tmp,-1
2148
	vperm		$inptail,$inptail,$tmp,$inpperm
2149
	vsel		$inout,$inout,$output,$inptail
2150

2151
	subi		r11,$out,17
2152
	subi		$out,$out,16
2153
	mtctr		$len
2154
	li		$len,16
2155
Loop_xts_enc_steal:
2156
	lbzu		r0,1(r11)
2157
	stb		r0,16(r11)
2158
	bdnz		Loop_xts_enc_steal
2159

2160
	mtctr		$rounds
2161
	b		Loop_xts_enc			# one more time...
2162

2163
Lxts_enc_done:
2164
	${UCMP}i	$ivp,0
2165
	beq		Lxts_enc_ret
2166

2167
	vsrab		$tmp,$tweak,$seven		# next tweak value
2168
	vaddubm		$tweak,$tweak,$tweak
2169
	vsldoi		$tmp,$tmp,$tmp,15
2170
	vand		$tmp,$tmp,$eighty7
2171
	vxor		$tweak,$tweak,$tmp
2172

2173
	le?vperm	$tweak,$tweak,$tweak,$leperm
2174
	stvx_u		$tweak,0,$ivp
2175

2176
Lxts_enc_ret:
2177
	mtspr		256,r12				# restore vrsave
2178
	li		r3,0
2179
	blr
2180
	.long		0
2181
	.byte		0,12,0x04,0,0x80,6,6,0
2182
	.long		0
2183
.size	.${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
2184

2185
.globl	.${prefix}_xts_decrypt
2186
	mr		$inp,r3				# reassign
2187
	li		r3,-1
2188
	${UCMP}i	$len,16
2189
	bltlr-
2190

2191
	lis		r0,0xfff8
2192
	mfspr		r12,256				# save vrsave
2193
	li		r11,0
2194
	mtspr		256,r0
2195

2196
	andi.		r0,$len,15
2197
	neg		r0,r0
2198
	andi.		r0,r0,16
2199
	sub		$len,$len,r0
2200

2201
	vspltisb	$seven,0x07			# 0x070707..07
2202
	le?lvsl		$leperm,r11,r11
2203
	le?vspltisb	$tmp,0x0f
2204
	le?vxor		$leperm,$leperm,$seven
2205

2206
	li		$idx,15
2207
	lvx		$tweak,0,$ivp			# load [unaligned] iv
2208
	lvsl		$inpperm,0,$ivp
2209
	lvx		$inptail,$idx,$ivp
2210
	le?vxor		$inpperm,$inpperm,$tmp
2211
	vperm		$tweak,$tweak,$inptail,$inpperm
2212

2213
	neg		r11,$inp
2214
	lvsr		$inpperm,0,r11			# prepare for unaligned load
2215
	lvx		$inout,0,$inp
2216
	addi		$inp,$inp,15			# 15 is not typo
2217
	le?vxor		$inpperm,$inpperm,$tmp
2218

2219
	${UCMP}i	$key2,0				# key2==NULL?
2220
	beq		Lxts_dec_no_key2
2221

2222
	?lvsl		$keyperm,0,$key2		# prepare for unaligned key
2223
	lwz		$rounds,240($key2)
2224
	srwi		$rounds,$rounds,1
2225
	subi		$rounds,$rounds,1
2226
	li		$idx,16
2227

2228
	lvx		$rndkey0,0,$key2
2229
	lvx		$rndkey1,$idx,$key2
2230
	addi		$idx,$idx,16
2231
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2232
	vxor		$tweak,$tweak,$rndkey0
2233
	lvx		$rndkey0,$idx,$key2
2234
	addi		$idx,$idx,16
2235
	mtctr		$rounds
2236

2237
Ltweak_xts_dec:
2238
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
2239
	vcipher		$tweak,$tweak,$rndkey1
2240
	lvx		$rndkey1,$idx,$key2
2241
	addi		$idx,$idx,16
2242
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2243
	vcipher		$tweak,$tweak,$rndkey0
2244
	lvx		$rndkey0,$idx,$key2
2245
	addi		$idx,$idx,16
2246
	bdnz		Ltweak_xts_dec
2247

2248
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
2249
	vcipher		$tweak,$tweak,$rndkey1
2250
	lvx		$rndkey1,$idx,$key2
2251
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2252
	vcipherlast	$tweak,$tweak,$rndkey0
2253

2254
	li		$ivp,0				# don't chain the tweak
2255
	b		Lxts_dec
2256

2257
Lxts_dec_no_key2:
2258
	neg		$idx,$len
2259
	andi.		$idx,$idx,15
2260
	add		$len,$len,$idx			# in "tweak chaining"
2261
							# mode only complete
2262
							# blocks are processed
2263
Lxts_dec:
2264
	lvx		$inptail,0,$inp
2265
	addi		$inp,$inp,16
2266

2267
	?lvsl		$keyperm,0,$key1		# prepare for unaligned key
2268
	lwz		$rounds,240($key1)
2269
	srwi		$rounds,$rounds,1
2270
	subi		$rounds,$rounds,1
2271
	li		$idx,16
2272

2273
	vslb		$eighty7,$seven,$seven		# 0x808080..80
2274
	vor		$eighty7,$eighty7,$seven	# 0x878787..87
2275
	vspltisb	$tmp,1				# 0x010101..01
2276
	vsldoi		$eighty7,$eighty7,$tmp,15	# 0x870101..01
2277

2278
	${UCMP}i	$len,96
2279
	bge		_aesp8_xts_decrypt6x
2280

2281
	lvx		$rndkey0,0,$key1
2282
	lvx		$rndkey1,$idx,$key1
2283
	addi		$idx,$idx,16
2284
	vperm		$inout,$inout,$inptail,$inpperm
2285
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2286
	vxor		$inout,$inout,$tweak
2287
	vxor		$inout,$inout,$rndkey0
2288
	lvx		$rndkey0,$idx,$key1
2289
	addi		$idx,$idx,16
2290
	mtctr		$rounds
2291

2292
	${UCMP}i	$len,16
2293
	blt		Ltail_xts_dec
2294
	be?b		Loop_xts_dec
2295

2296
.align	5
2297
Loop_xts_dec:
2298
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
2299
	vncipher	$inout,$inout,$rndkey1
2300
	lvx		$rndkey1,$idx,$key1
2301
	addi		$idx,$idx,16
2302
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2303
	vncipher	$inout,$inout,$rndkey0
2304
	lvx		$rndkey0,$idx,$key1
2305
	addi		$idx,$idx,16
2306
	bdnz		Loop_xts_dec
2307

2308
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
2309
	vncipher	$inout,$inout,$rndkey1
2310
	lvx		$rndkey1,$idx,$key1
2311
	li		$idx,16
2312
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2313
	vxor		$rndkey0,$rndkey0,$tweak
2314
	vncipherlast	$output,$inout,$rndkey0
2315

2316
	le?vperm	$tmp,$output,$output,$leperm
2317
	be?nop
2318
	le?stvx_u	$tmp,0,$out
2319
	be?stvx_u	$output,0,$out
2320
	addi		$out,$out,16
2321

2322
	subic.		$len,$len,16
2323
	beq		Lxts_dec_done
2324

2325
	vmr		$inout,$inptail
2326
	lvx		$inptail,0,$inp
2327
	addi		$inp,$inp,16
2328
	lvx		$rndkey0,0,$key1
2329
	lvx		$rndkey1,$idx,$key1
2330
	addi		$idx,$idx,16
2331

2332
	vsrab		$tmp,$tweak,$seven		# next tweak value
2333
	vaddubm		$tweak,$tweak,$tweak
2334
	vsldoi		$tmp,$tmp,$tmp,15
2335
	vand		$tmp,$tmp,$eighty7
2336
	vxor		$tweak,$tweak,$tmp
2337

2338
	vperm		$inout,$inout,$inptail,$inpperm
2339
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2340
	vxor		$inout,$inout,$tweak
2341
	vxor		$inout,$inout,$rndkey0
2342
	lvx		$rndkey0,$idx,$key1
2343
	addi		$idx,$idx,16
2344

2345
	mtctr		$rounds
2346
	${UCMP}i	$len,16
2347
	bge		Loop_xts_dec
2348

2349
Ltail_xts_dec:
2350
	vsrab		$tmp,$tweak,$seven		# next tweak value
2351
	vaddubm		$tweak1,$tweak,$tweak
2352
	vsldoi		$tmp,$tmp,$tmp,15
2353
	vand		$tmp,$tmp,$eighty7
2354
	vxor		$tweak1,$tweak1,$tmp
2355

2356
	subi		$inp,$inp,16
2357
	add		$inp,$inp,$len
2358

2359
	vxor		$inout,$inout,$tweak		# :-(
2360
	vxor		$inout,$inout,$tweak1		# :-)
2361

2362
Loop_xts_dec_short:
2363
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
2364
	vncipher	$inout,$inout,$rndkey1
2365
	lvx		$rndkey1,$idx,$key1
2366
	addi		$idx,$idx,16
2367
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2368
	vncipher	$inout,$inout,$rndkey0
2369
	lvx		$rndkey0,$idx,$key1
2370
	addi		$idx,$idx,16
2371
	bdnz		Loop_xts_dec_short
2372

2373
	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
2374
	vncipher	$inout,$inout,$rndkey1
2375
	lvx		$rndkey1,$idx,$key1
2376
	li		$idx,16
2377
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2378
	vxor		$rndkey0,$rndkey0,$tweak1
2379
	vncipherlast	$output,$inout,$rndkey0
2380

2381
	le?vperm	$tmp,$output,$output,$leperm
2382
	be?nop
2383
	le?stvx_u	$tmp,0,$out
2384
	be?stvx_u	$output,0,$out
2385

2386
	vmr		$inout,$inptail
2387
	lvx		$inptail,0,$inp
2388
	#addi		$inp,$inp,16
2389
	lvx		$rndkey0,0,$key1
2390
	lvx		$rndkey1,$idx,$key1
2391
	addi		$idx,$idx,16
2392
	vperm		$inout,$inout,$inptail,$inpperm
2393
	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
2394

2395
	lvsr		$inpperm,0,$len			# $inpperm is no longer needed
2396
	vxor		$inptail,$inptail,$inptail	# $inptail is no longer needed
2397
	vspltisb	$tmp,-1
2398
	vperm		$inptail,$inptail,$tmp,$inpperm
2399
	vsel		$inout,$inout,$output,$inptail
2400

2401
	vxor		$rndkey0,$rndkey0,$tweak
2402
	vxor		$inout,$inout,$rndkey0
2403
	lvx		$rndkey0,$idx,$key1
2404
	addi		$idx,$idx,16
2405

2406
	subi		r11,$out,1
2407
	mtctr		$len
2408
	li		$len,16
2409
Loop_xts_dec_steal:
2410
	lbzu		r0,1(r11)
2411
	stb		r0,16(r11)
2412
	bdnz		Loop_xts_dec_steal
2413

2414
	mtctr		$rounds
2415
	b		Loop_xts_dec			# one more time...
2416

2417
Lxts_dec_done:
2418
	${UCMP}i	$ivp,0
2419
	beq		Lxts_dec_ret
2420

2421
	vsrab		$tmp,$tweak,$seven		# next tweak value
2422
	vaddubm		$tweak,$tweak,$tweak
2423
	vsldoi		$tmp,$tmp,$tmp,15
2424
	vand		$tmp,$tmp,$eighty7
2425
	vxor		$tweak,$tweak,$tmp
2426

2427
	le?vperm	$tweak,$tweak,$tweak,$leperm
2428
	stvx_u		$tweak,0,$ivp
2429

2430
Lxts_dec_ret:
2431
	mtspr		256,r12				# restore vrsave
2432
	li		r3,0
2433
	blr
2434
	.long		0
2435
	.byte		0,12,0x04,0,0x80,6,6,0
2436
	.long		0
2437
.size	.${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
2438
___
2439
#########################################################################
2440
{{	# Optimized XTS procedures					#
2441
my $key_=$key2;
2442
my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31));
2443
    $x00=0 if ($flavour =~ /osx/);
2444
my ($in0,  $in1,  $in2,  $in3,  $in4,  $in5 )=map("v$_",(0..5));
2445
my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16));
2446
my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22));
2447
my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
2448
			# v26-v31 last 6 round keys
2449
my ($keyperm)=($out0);	# aliases with "caller", redundant assignment
2450
my $taillen=$x70;
2451

2452
$code.=<<___;
2453
.align	5
2454
_aesp8_xts_encrypt6x:
2455
	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
2456
	mflr		r11
2457
	li		r7,`$FRAME+8*16+15`
2458
	li		r3,`$FRAME+8*16+31`
2459
	$PUSH		r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
2460
	stvx		v20,r7,$sp		# ABI says so
2461
	addi		r7,r7,32
2462
	stvx		v21,r3,$sp
2463
	addi		r3,r3,32
2464
	stvx		v22,r7,$sp
2465
	addi		r7,r7,32
2466
	stvx		v23,r3,$sp
2467
	addi		r3,r3,32
2468
	stvx		v24,r7,$sp
2469
	addi		r7,r7,32
2470
	stvx		v25,r3,$sp
2471
	addi		r3,r3,32
2472
	stvx		v26,r7,$sp
2473
	addi		r7,r7,32
2474
	stvx		v27,r3,$sp
2475
	addi		r3,r3,32
2476
	stvx		v28,r7,$sp
2477
	addi		r7,r7,32
2478
	stvx		v29,r3,$sp
2479
	addi		r3,r3,32
2480
	stvx		v30,r7,$sp
2481
	stvx		v31,r3,$sp
2482
	li		r0,-1
2483
	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
2484
	li		$x10,0x10
2485
	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
2486
	li		$x20,0x20
2487
	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
2488
	li		$x30,0x30
2489
	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
2490
	li		$x40,0x40
2491
	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
2492
	li		$x50,0x50
2493
	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
2494
	li		$x60,0x60
2495
	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
2496
	li		$x70,0x70
2497
	mtspr		256,r0
2498

2499
	xxlor		2, 32+$eighty7, 32+$eighty7
2500
	vsldoi		$eighty7,$tmp,$eighty7,1        # 0x010101..87
2501
	xxlor		1, 32+$eighty7, 32+$eighty7
2502

2503
	# Load XOR Lconsts.
2504
	mr		$x70, r6
2505
	bl		Lconsts
2506
	lxvw4x		0, $x40, r6		# load XOR contents
2507
	mr		r6, $x70
2508
	li		$x70,0x70
2509

2510
	subi		$rounds,$rounds,3	# -4 in total
2511

2512
	lvx		$rndkey0,$x00,$key1	# load key schedule
2513
	lvx		v30,$x10,$key1
2514
	addi		$key1,$key1,0x20
2515
	lvx		v31,$x00,$key1
2516
	?vperm		$rndkey0,$rndkey0,v30,$keyperm
2517
	addi		$key_,$sp,$FRAME+15
2518
	mtctr		$rounds
2519

2520
Load_xts_enc_key:
2521
	?vperm		v24,v30,v31,$keyperm
2522
	lvx		v30,$x10,$key1
2523
	addi		$key1,$key1,0x20
2524
	stvx		v24,$x00,$key_		# off-load round[1]
2525
	?vperm		v25,v31,v30,$keyperm
2526
	lvx		v31,$x00,$key1
2527
	stvx		v25,$x10,$key_		# off-load round[2]
2528
	addi		$key_,$key_,0x20
2529
	bdnz		Load_xts_enc_key
2530

2531
	lvx		v26,$x10,$key1
2532
	?vperm		v24,v30,v31,$keyperm
2533
	lvx		v27,$x20,$key1
2534
	stvx		v24,$x00,$key_		# off-load round[3]
2535
	?vperm		v25,v31,v26,$keyperm
2536
	lvx		v28,$x30,$key1
2537
	stvx		v25,$x10,$key_		# off-load round[4]
2538
	addi		$key_,$sp,$FRAME+15	# rewind $key_
2539
	?vperm		v26,v26,v27,$keyperm
2540
	lvx		v29,$x40,$key1
2541
	?vperm		v27,v27,v28,$keyperm
2542
	lvx		v30,$x50,$key1
2543
	?vperm		v28,v28,v29,$keyperm
2544
	lvx		v31,$x60,$key1
2545
	?vperm		v29,v29,v30,$keyperm
2546
	lvx		$twk5,$x70,$key1	# borrow $twk5
2547
	?vperm		v30,v30,v31,$keyperm
2548
	lvx		v24,$x00,$key_		# pre-load round[1]
2549
	?vperm		v31,v31,$twk5,$keyperm
2550
	lvx		v25,$x10,$key_		# pre-load round[2]
2551

2552
	# Switch to use the following codes with 0x010101..87 to generate tweak.
2553
	#     eighty7 = 0x010101..87
2554
	# vsrab         tmp, tweak, seven       # next tweak value, right shift 7 bits
2555
	# vand          tmp, tmp, eighty7       # last byte with carry
2556
	# vaddubm       tweak, tweak, tweak     # left shift 1 bit (x2)
2557
	# xxlor         vsx, 0, 0
2558
	# vpermxor      tweak, tweak, tmp, vsx
2559

2560
	 vperm		$in0,$inout,$inptail,$inpperm
2561
	 subi		$inp,$inp,31		# undo "caller"
2562
	vxor		$twk0,$tweak,$rndkey0
2563
	vsrab		$tmp,$tweak,$seven	# next tweak value
2564
	vaddubm		$tweak,$tweak,$tweak
2565
	vand		$tmp,$tmp,$eighty7
2566
	 vxor		$out0,$in0,$twk0
2567
	xxlor		32+$in1, 0, 0
2568
	vpermxor	$tweak, $tweak, $tmp, $in1
2569

2570
	 lvx_u		$in1,$x10,$inp
2571
	vxor		$twk1,$tweak,$rndkey0
2572
	vsrab		$tmp,$tweak,$seven	# next tweak value
2573
	vaddubm		$tweak,$tweak,$tweak
2574
	 le?vperm	$in1,$in1,$in1,$leperm
2575
	vand		$tmp,$tmp,$eighty7
2576
	 vxor		$out1,$in1,$twk1
2577
	xxlor		32+$in2, 0, 0
2578
	vpermxor	$tweak, $tweak, $tmp, $in2
2579

2580
	 lvx_u		$in2,$x20,$inp
2581
	 andi.		$taillen,$len,15
2582
	vxor		$twk2,$tweak,$rndkey0
2583
	vsrab		$tmp,$tweak,$seven	# next tweak value
2584
	vaddubm		$tweak,$tweak,$tweak
2585
	 le?vperm	$in2,$in2,$in2,$leperm
2586
	vand		$tmp,$tmp,$eighty7
2587
	 vxor		$out2,$in2,$twk2
2588
	xxlor		32+$in3, 0, 0
2589
	vpermxor	$tweak, $tweak, $tmp, $in3
2590

2591
	 lvx_u		$in3,$x30,$inp
2592
	 sub		$len,$len,$taillen
2593
	vxor		$twk3,$tweak,$rndkey0
2594
	vsrab		$tmp,$tweak,$seven	# next tweak value
2595
	vaddubm		$tweak,$tweak,$tweak
2596
	 le?vperm	$in3,$in3,$in3,$leperm
2597
	vand		$tmp,$tmp,$eighty7
2598
	 vxor		$out3,$in3,$twk3
2599
	xxlor		32+$in4, 0, 0
2600
	vpermxor	$tweak, $tweak, $tmp, $in4
2601

2602
	 lvx_u		$in4,$x40,$inp
2603
	 subi		$len,$len,0x60
2604
	vxor		$twk4,$tweak,$rndkey0
2605
	vsrab		$tmp,$tweak,$seven	# next tweak value
2606
	vaddubm		$tweak,$tweak,$tweak
2607
	 le?vperm	$in4,$in4,$in4,$leperm
2608
	vand		$tmp,$tmp,$eighty7
2609
	 vxor		$out4,$in4,$twk4
2610
	xxlor		32+$in5, 0, 0
2611
	vpermxor	$tweak, $tweak, $tmp, $in5
2612

2613
	 lvx_u		$in5,$x50,$inp
2614
	 addi		$inp,$inp,0x60
2615
	vxor		$twk5,$tweak,$rndkey0
2616
	vsrab		$tmp,$tweak,$seven	# next tweak value
2617
	vaddubm		$tweak,$tweak,$tweak
2618
	 le?vperm	$in5,$in5,$in5,$leperm
2619
	vand		$tmp,$tmp,$eighty7
2620
	 vxor		$out5,$in5,$twk5
2621
	xxlor		32+$in0, 0, 0
2622
	vpermxor	$tweak, $tweak, $tmp, $in0
2623

2624
	vxor		v31,v31,$rndkey0
2625
	mtctr		$rounds
2626
	b		Loop_xts_enc6x
2627

2628
.align	5
2629
Loop_xts_enc6x:
2630
	vcipher		$out0,$out0,v24
2631
	vcipher		$out1,$out1,v24
2632
	vcipher		$out2,$out2,v24
2633
	vcipher		$out3,$out3,v24
2634
	vcipher		$out4,$out4,v24
2635
	vcipher		$out5,$out5,v24
2636
	lvx		v24,$x20,$key_		# round[3]
2637
	addi		$key_,$key_,0x20
2638

2639
	vcipher		$out0,$out0,v25
2640
	vcipher		$out1,$out1,v25
2641
	vcipher		$out2,$out2,v25
2642
	vcipher		$out3,$out3,v25
2643
	vcipher		$out4,$out4,v25
2644
	vcipher		$out5,$out5,v25
2645
	lvx		v25,$x10,$key_		# round[4]
2646
	bdnz		Loop_xts_enc6x
2647

2648
	xxlor		32+$eighty7, 1, 1	# 0x010101..87
2649

2650
	subic		$len,$len,96		# $len-=96
2651
	 vxor		$in0,$twk0,v31		# xor with last round key
2652
	vcipher		$out0,$out0,v24
2653
	vcipher		$out1,$out1,v24
2654
	 vsrab		$tmp,$tweak,$seven	# next tweak value
2655
	 vxor		$twk0,$tweak,$rndkey0
2656
	 vaddubm	$tweak,$tweak,$tweak
2657
	vcipher		$out2,$out2,v24
2658
	vcipher		$out3,$out3,v24
2659
	vcipher		$out4,$out4,v24
2660
	vcipher		$out5,$out5,v24
2661

2662
	subfe.		r0,r0,r0		# borrow?-1:0
2663
	 vand		$tmp,$tmp,$eighty7
2664
	vcipher		$out0,$out0,v25
2665
	vcipher		$out1,$out1,v25
2666
	 xxlor		32+$in1, 0, 0
2667
	 vpermxor	$tweak, $tweak, $tmp, $in1
2668
	vcipher		$out2,$out2,v25
2669
	vcipher		$out3,$out3,v25
2670
	 vxor		$in1,$twk1,v31
2671
	 vsrab		$tmp,$tweak,$seven	# next tweak value
2672
	 vxor		$twk1,$tweak,$rndkey0
2673
	vcipher		$out4,$out4,v25
2674
	vcipher		$out5,$out5,v25
2675

2676
	and		r0,r0,$len
2677
	 vaddubm	$tweak,$tweak,$tweak
2678
	vcipher		$out0,$out0,v26
2679
	vcipher		$out1,$out1,v26
2680
	 vand		$tmp,$tmp,$eighty7
2681
	vcipher		$out2,$out2,v26
2682
	vcipher		$out3,$out3,v26
2683
	 xxlor		32+$in2, 0, 0
2684
	 vpermxor	$tweak, $tweak, $tmp, $in2
2685
	vcipher		$out4,$out4,v26
2686
	vcipher		$out5,$out5,v26
2687

2688
	add		$inp,$inp,r0		# $inp is adjusted in such
2689
						# way that at exit from the
2690
						# loop inX-in5 are loaded
2691
						# with last "words"
2692
	 vxor		$in2,$twk2,v31
2693
	 vsrab		$tmp,$tweak,$seven	# next tweak value
2694
	 vxor		$twk2,$tweak,$rndkey0
2695
	 vaddubm	$tweak,$tweak,$tweak
2696
	vcipher		$out0,$out0,v27
2697
	vcipher		$out1,$out1,v27
2698
	vcipher		$out2,$out2,v27
2699
	vcipher		$out3,$out3,v27
2700
	 vand		$tmp,$tmp,$eighty7
2701
	vcipher		$out4,$out4,v27
2702
	vcipher		$out5,$out5,v27
2703

2704
	addi		$key_,$sp,$FRAME+15	# rewind $key_
2705
	 xxlor		32+$in3, 0, 0
2706
	 vpermxor	$tweak, $tweak, $tmp, $in3
2707
	vcipher		$out0,$out0,v28
2708
	vcipher		$out1,$out1,v28
2709
	 vxor		$in3,$twk3,v31
2710
	 vsrab		$tmp,$tweak,$seven	# next tweak value
2711
	 vxor		$twk3,$tweak,$rndkey0
2712
	vcipher		$out2,$out2,v28
2713
	vcipher		$out3,$out3,v28
2714
	 vaddubm	$tweak,$tweak,$tweak
2715
	vcipher		$out4,$out4,v28
2716
	vcipher		$out5,$out5,v28
2717
	lvx		v24,$x00,$key_		# re-pre-load round[1]
2718
	 vand		$tmp,$tmp,$eighty7
2719

2720
	vcipher		$out0,$out0,v29
2721
	vcipher		$out1,$out1,v29
2722
	 xxlor		32+$in4, 0, 0
2723
	 vpermxor	$tweak, $tweak, $tmp, $in4
2724
	vcipher		$out2,$out2,v29
2725
	vcipher		$out3,$out3,v29
2726
	 vxor		$in4,$twk4,v31
2727
	 vsrab		$tmp,$tweak,$seven	# next tweak value
2728
	 vxor		$twk4,$tweak,$rndkey0
2729
	vcipher		$out4,$out4,v29
2730
	vcipher		$out5,$out5,v29
2731
	lvx		v25,$x10,$key_		# re-pre-load round[2]
2732
	 vaddubm	$tweak,$tweak,$tweak
2733

2734
	vcipher		$out0,$out0,v30
2735
	vcipher		$out1,$out1,v30
2736
	 vand		$tmp,$tmp,$eighty7
2737
	vcipher		$out2,$out2,v30
2738
	vcipher		$out3,$out3,v30
2739
	 xxlor		32+$in5, 0, 0
2740
	 vpermxor	$tweak, $tweak, $tmp, $in5
2741
	vcipher		$out4,$out4,v30
2742
	vcipher		$out5,$out5,v30
2743
	 vxor		$in5,$twk5,v31
2744
	 vsrab		$tmp,$tweak,$seven	# next tweak value
2745
	 vxor		$twk5,$tweak,$rndkey0
2746

2747
	vcipherlast	$out0,$out0,$in0
2748
	 lvx_u		$in0,$x00,$inp		# load next input block
2749
	 vaddubm	$tweak,$tweak,$tweak
2750
	vcipherlast	$out1,$out1,$in1
2751
	 lvx_u		$in1,$x10,$inp
2752
	vcipherlast	$out2,$out2,$in2
2753
	 le?vperm	$in0,$in0,$in0,$leperm
2754
	 lvx_u		$in2,$x20,$inp
2755
	 vand		$tmp,$tmp,$eighty7
2756
	vcipherlast	$out3,$out3,$in3
2757
	 le?vperm	$in1,$in1,$in1,$leperm
2758
	 lvx_u		$in3,$x30,$inp
2759
	vcipherlast	$out4,$out4,$in4
2760
	 le?vperm	$in2,$in2,$in2,$leperm
2761
	 lvx_u		$in4,$x40,$inp
2762
	 xxlor		10, 32+$in0, 32+$in0
2763
	 xxlor		32+$in0, 0, 0
2764
	 vpermxor	$tweak, $tweak, $tmp, $in0
2765
	 xxlor		32+$in0, 10, 10
2766
	vcipherlast	$tmp,$out5,$in5		# last block might be needed
2767
						# in stealing mode
2768
	 le?vperm	$in3,$in3,$in3,$leperm
2769
	 lvx_u		$in5,$x50,$inp
2770
	 addi		$inp,$inp,0x60
2771
	 le?vperm	$in4,$in4,$in4,$leperm
2772
	 le?vperm	$in5,$in5,$in5,$leperm
2773

2774
	le?vperm	$out0,$out0,$out0,$leperm
2775
	le?vperm	$out1,$out1,$out1,$leperm
2776
	stvx_u		$out0,$x00,$out		# store output
2777
	 vxor		$out0,$in0,$twk0
2778
	le?vperm	$out2,$out2,$out2,$leperm
2779
	stvx_u		$out1,$x10,$out
2780
	 vxor		$out1,$in1,$twk1
2781
	le?vperm	$out3,$out3,$out3,$leperm
2782
	stvx_u		$out2,$x20,$out
2783
	 vxor		$out2,$in2,$twk2
2784
	le?vperm	$out4,$out4,$out4,$leperm
2785
	stvx_u		$out3,$x30,$out
2786
	 vxor		$out3,$in3,$twk3
2787
	le?vperm	$out5,$tmp,$tmp,$leperm
2788
	stvx_u		$out4,$x40,$out
2789
	 vxor		$out4,$in4,$twk4
2790
	le?stvx_u	$out5,$x50,$out
2791
	be?stvx_u	$tmp, $x50,$out
2792
	 vxor		$out5,$in5,$twk5
2793
	addi		$out,$out,0x60
2794

2795
	mtctr		$rounds
2796
	beq		Loop_xts_enc6x		# did $len-=96 borrow?
2797

2798
	xxlor		32+$eighty7, 2, 2	# 0x010101..87
2799

2800
	addic.		$len,$len,0x60
2801
	beq		Lxts_enc6x_zero
2802
	cmpwi		$len,0x20
2803
	blt		Lxts_enc6x_one
2804
	nop
2805
	beq		Lxts_enc6x_two
2806
	cmpwi		$len,0x40
2807
	blt		Lxts_enc6x_three
2808
	nop
2809
	beq		Lxts_enc6x_four
2810

2811
Lxts_enc6x_five:
2812
	vxor		$out0,$in1,$twk0
2813
	vxor		$out1,$in2,$twk1
2814
	vxor		$out2,$in3,$twk2
2815
	vxor		$out3,$in4,$twk3
2816
	vxor		$out4,$in5,$twk4
2817

2818
	bl		_aesp8_xts_enc5x
2819

2820
	le?vperm	$out0,$out0,$out0,$leperm
2821
	vmr		$twk0,$twk5		# unused tweak
2822
	le?vperm	$out1,$out1,$out1,$leperm
2823
	stvx_u		$out0,$x00,$out		# store output
2824
	le?vperm	$out2,$out2,$out2,$leperm
2825
	stvx_u		$out1,$x10,$out
2826
	le?vperm	$out3,$out3,$out3,$leperm
2827
	stvx_u		$out2,$x20,$out
2828
	vxor		$tmp,$out4,$twk5	# last block prep for stealing
2829
	le?vperm	$out4,$out4,$out4,$leperm
2830
	stvx_u		$out3,$x30,$out
2831
	stvx_u		$out4,$x40,$out
2832
	addi		$out,$out,0x50
2833
	bne		Lxts_enc6x_steal
2834
	b		Lxts_enc6x_done
2835

2836
.align	4
2837
Lxts_enc6x_four:
2838
	vxor		$out0,$in2,$twk0
2839
	vxor		$out1,$in3,$twk1
2840
	vxor		$out2,$in4,$twk2
2841
	vxor		$out3,$in5,$twk3
2842
	vxor		$out4,$out4,$out4
2843

2844
	bl		_aesp8_xts_enc5x
2845

2846
	le?vperm	$out0,$out0,$out0,$leperm
2847
	vmr		$twk0,$twk4		# unused tweak
2848
	le?vperm	$out1,$out1,$out1,$leperm
2849
	stvx_u		$out0,$x00,$out		# store output
2850
	le?vperm	$out2,$out2,$out2,$leperm
2851
	stvx_u		$out1,$x10,$out
2852
	vxor		$tmp,$out3,$twk4	# last block prep for stealing
2853
	le?vperm	$out3,$out3,$out3,$leperm
2854
	stvx_u		$out2,$x20,$out
2855
	stvx_u		$out3,$x30,$out
2856
	addi		$out,$out,0x40
2857
	bne		Lxts_enc6x_steal
2858
	b		Lxts_enc6x_done
2859

2860
.align	4
2861
Lxts_enc6x_three:
2862
	vxor		$out0,$in3,$twk0
2863
	vxor		$out1,$in4,$twk1
2864
	vxor		$out2,$in5,$twk2
2865
	vxor		$out3,$out3,$out3
2866
	vxor		$out4,$out4,$out4
2867

2868
	bl		_aesp8_xts_enc5x
2869

2870
	le?vperm	$out0,$out0,$out0,$leperm
2871
	vmr		$twk0,$twk3		# unused tweak
2872
	le?vperm	$out1,$out1,$out1,$leperm
2873
	stvx_u		$out0,$x00,$out		# store output
2874
	vxor		$tmp,$out2,$twk3	# last block prep for stealing
2875
	le?vperm	$out2,$out2,$out2,$leperm
2876
	stvx_u		$out1,$x10,$out
2877
	stvx_u		$out2,$x20,$out
2878
	addi		$out,$out,0x30
2879
	bne		Lxts_enc6x_steal
2880
	b		Lxts_enc6x_done
2881

2882
.align	4
2883
Lxts_enc6x_two:
2884
	vxor		$out0,$in4,$twk0
2885
	vxor		$out1,$in5,$twk1
2886
	vxor		$out2,$out2,$out2
2887
	vxor		$out3,$out3,$out3
2888
	vxor		$out4,$out4,$out4
2889

2890
	bl		_aesp8_xts_enc5x
2891

2892
	le?vperm	$out0,$out0,$out0,$leperm
2893
	vmr		$twk0,$twk2		# unused tweak
2894
	vxor		$tmp,$out1,$twk2	# last block prep for stealing
2895
	le?vperm	$out1,$out1,$out1,$leperm
2896
	stvx_u		$out0,$x00,$out		# store output
2897
	stvx_u		$out1,$x10,$out
2898
	addi		$out,$out,0x20
2899
	bne		Lxts_enc6x_steal
2900
	b		Lxts_enc6x_done
2901

2902
.align	4
2903
Lxts_enc6x_one:
2904
	vxor		$out0,$in5,$twk0
2905
	nop
2906
Loop_xts_enc1x:
2907
	vcipher		$out0,$out0,v24
2908
	lvx		v24,$x20,$key_		# round[3]
2909
	addi		$key_,$key_,0x20
2910

2911
	vcipher		$out0,$out0,v25
2912
	lvx		v25,$x10,$key_		# round[4]
2913
	bdnz		Loop_xts_enc1x
2914

2915
	add		$inp,$inp,$taillen
2916
	cmpwi		$taillen,0
2917
	vcipher		$out0,$out0,v24
2918

2919
	subi		$inp,$inp,16
2920
	vcipher		$out0,$out0,v25
2921

2922
	lvsr		$inpperm,0,$taillen
2923
	vcipher		$out0,$out0,v26
2924

2925
	lvx_u		$in0,0,$inp
2926
	vcipher		$out0,$out0,v27
2927

2928
	addi		$key_,$sp,$FRAME+15	# rewind $key_
2929
	vcipher		$out0,$out0,v28
2930
	lvx		v24,$x00,$key_		# re-pre-load round[1]
2931

2932
	vcipher		$out0,$out0,v29
2933
	lvx		v25,$x10,$key_		# re-pre-load round[2]
2934
	 vxor		$twk0,$twk0,v31
2935

2936
	le?vperm	$in0,$in0,$in0,$leperm
2937
	vcipher		$out0,$out0,v30
2938

2939
	vperm		$in0,$in0,$in0,$inpperm
2940
	vcipherlast	$out0,$out0,$twk0
2941

2942
	vmr		$twk0,$twk1		# unused tweak
2943
	vxor		$tmp,$out0,$twk1	# last block prep for stealing
2944
	le?vperm	$out0,$out0,$out0,$leperm
2945
	stvx_u		$out0,$x00,$out		# store output
2946
	addi		$out,$out,0x10
2947
	bne		Lxts_enc6x_steal
2948
	b		Lxts_enc6x_done
2949

2950
.align	4
2951
Lxts_enc6x_zero:
2952
	cmpwi		$taillen,0
2953
	beq		Lxts_enc6x_done
2954

2955
	add		$inp,$inp,$taillen
2956
	subi		$inp,$inp,16
2957
	lvx_u		$in0,0,$inp
2958
	lvsr		$inpperm,0,$taillen	# $in5 is no more
2959
	le?vperm	$in0,$in0,$in0,$leperm
2960
	vperm		$in0,$in0,$in0,$inpperm
2961
	vxor		$tmp,$tmp,$twk0
2962
Lxts_enc6x_steal:
2963
	vxor		$in0,$in0,$twk0
2964
	vxor		$out0,$out0,$out0
2965
	vspltisb	$out1,-1
2966
	vperm		$out0,$out0,$out1,$inpperm
2967
	vsel		$out0,$in0,$tmp,$out0	# $tmp is last block, remember?
2968

2969
	subi		r30,$out,17
2970
	subi		$out,$out,16
2971
	mtctr		$taillen
2972
Loop_xts_enc6x_steal:
2973
	lbzu		r0,1(r30)
2974
	stb		r0,16(r30)
2975
	bdnz		Loop_xts_enc6x_steal
2976

2977
	li		$taillen,0
2978
	mtctr		$rounds
2979
	b		Loop_xts_enc1x		# one more time...
2980

2981
.align	4
2982
Lxts_enc6x_done:
2983
	${UCMP}i	$ivp,0
2984
	beq		Lxts_enc6x_ret
2985

2986
	vxor		$tweak,$twk0,$rndkey0
2987
	le?vperm	$tweak,$tweak,$tweak,$leperm
2988
	stvx_u		$tweak,0,$ivp
2989

2990
Lxts_enc6x_ret:
2991
	mtlr		r11
2992
	li		r10,`$FRAME+15`
2993
	li		r11,`$FRAME+31`
2994
	stvx		$seven,r10,$sp		# wipe copies of round keys
2995
	addi		r10,r10,32
2996
	stvx		$seven,r11,$sp
2997
	addi		r11,r11,32
2998
	stvx		$seven,r10,$sp
2999
	addi		r10,r10,32
3000
	stvx		$seven,r11,$sp
3001
	addi		r11,r11,32
3002
	stvx		$seven,r10,$sp
3003
	addi		r10,r10,32
3004
	stvx		$seven,r11,$sp
3005
	addi		r11,r11,32
3006
	stvx		$seven,r10,$sp
3007
	addi		r10,r10,32
3008
	stvx		$seven,r11,$sp
3009
	addi		r11,r11,32
3010

3011
	mtspr		256,$vrsave
3012
	lvx		v20,r10,$sp		# ABI says so
3013
	addi		r10,r10,32
3014
	lvx		v21,r11,$sp
3015
	addi		r11,r11,32
3016
	lvx		v22,r10,$sp
3017
	addi		r10,r10,32
3018
	lvx		v23,r11,$sp
3019
	addi		r11,r11,32
3020
	lvx		v24,r10,$sp
3021
	addi		r10,r10,32
3022
	lvx		v25,r11,$sp
3023
	addi		r11,r11,32
3024
	lvx		v26,r10,$sp
3025
	addi		r10,r10,32
3026
	lvx		v27,r11,$sp
3027
	addi		r11,r11,32
3028
	lvx		v28,r10,$sp
3029
	addi		r10,r10,32
3030
	lvx		v29,r11,$sp
3031
	addi		r11,r11,32
3032
	lvx		v30,r10,$sp
3033
	lvx		v31,r11,$sp
3034
	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
3035
	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
3036
	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
3037
	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
3038
	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
3039
	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
3040
	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
3041
	blr
3042
	.long		0
3043
	.byte		0,12,0x04,1,0x80,6,6,0
3044
	.long		0
3045

3046
.align	5
3047
_aesp8_xts_enc5x:
3048
	vcipher		$out0,$out0,v24
3049
	vcipher		$out1,$out1,v24
3050
	vcipher		$out2,$out2,v24
3051
	vcipher		$out3,$out3,v24
3052
	vcipher		$out4,$out4,v24
3053
	lvx		v24,$x20,$key_		# round[3]
3054
	addi		$key_,$key_,0x20
3055

3056
	vcipher		$out0,$out0,v25
3057
	vcipher		$out1,$out1,v25
3058
	vcipher		$out2,$out2,v25
3059
	vcipher		$out3,$out3,v25
3060
	vcipher		$out4,$out4,v25
3061
	lvx		v25,$x10,$key_		# round[4]
3062
	bdnz		_aesp8_xts_enc5x
3063

3064
	add		$inp,$inp,$taillen
3065
	cmpwi		$taillen,0
3066
	vcipher		$out0,$out0,v24
3067
	vcipher		$out1,$out1,v24
3068
	vcipher		$out2,$out2,v24
3069
	vcipher		$out3,$out3,v24
3070
	vcipher		$out4,$out4,v24
3071

3072
	subi		$inp,$inp,16
3073
	vcipher		$out0,$out0,v25
3074
	vcipher		$out1,$out1,v25
3075
	vcipher		$out2,$out2,v25
3076
	vcipher		$out3,$out3,v25
3077
	vcipher		$out4,$out4,v25
3078
	 vxor		$twk0,$twk0,v31
3079

3080
	vcipher		$out0,$out0,v26
3081
	lvsr		$inpperm,r0,$taillen	# $in5 is no more
3082
	vcipher		$out1,$out1,v26
3083
	vcipher		$out2,$out2,v26
3084
	vcipher		$out3,$out3,v26
3085
	vcipher		$out4,$out4,v26
3086
	 vxor		$in1,$twk1,v31
3087

3088
	vcipher		$out0,$out0,v27
3089
	lvx_u		$in0,0,$inp
3090
	vcipher		$out1,$out1,v27
3091
	vcipher		$out2,$out2,v27
3092
	vcipher		$out3,$out3,v27
3093
	vcipher		$out4,$out4,v27
3094
	 vxor		$in2,$twk2,v31
3095

3096
	addi		$key_,$sp,$FRAME+15	# rewind $key_
3097
	vcipher		$out0,$out0,v28
3098
	vcipher		$out1,$out1,v28
3099
	vcipher		$out2,$out2,v28
3100
	vcipher		$out3,$out3,v28
3101
	vcipher		$out4,$out4,v28
3102
	lvx		v24,$x00,$key_		# re-pre-load round[1]
3103
	 vxor		$in3,$twk3,v31
3104

3105
	vcipher		$out0,$out0,v29
3106
	le?vperm	$in0,$in0,$in0,$leperm
3107
	vcipher		$out1,$out1,v29
3108
	vcipher		$out2,$out2,v29
3109
	vcipher		$out3,$out3,v29
3110
	vcipher		$out4,$out4,v29
3111
	lvx		v25,$x10,$key_		# re-pre-load round[2]
3112
	 vxor		$in4,$twk4,v31
3113

3114
	vcipher		$out0,$out0,v30
3115
	vperm		$in0,$in0,$in0,$inpperm
3116
	vcipher		$out1,$out1,v30
3117
	vcipher		$out2,$out2,v30
3118
	vcipher		$out3,$out3,v30
3119
	vcipher		$out4,$out4,v30
3120

3121
	vcipherlast	$out0,$out0,$twk0
3122
	vcipherlast	$out1,$out1,$in1
3123
	vcipherlast	$out2,$out2,$in2
3124
	vcipherlast	$out3,$out3,$in3
3125
	vcipherlast	$out4,$out4,$in4
3126
	blr
3127
        .long   	0
3128
        .byte   	0,12,0x14,0,0,0,0,0
3129

3130
.align	5
3131
_aesp8_xts_decrypt6x:
3132
	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
3133
	mflr		r11
3134
	li		r7,`$FRAME+8*16+15`
3135
	li		r3,`$FRAME+8*16+31`
3136
	$PUSH		r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
3137
	stvx		v20,r7,$sp		# ABI says so
3138
	addi		r7,r7,32
3139
	stvx		v21,r3,$sp
3140
	addi		r3,r3,32
3141
	stvx		v22,r7,$sp
3142
	addi		r7,r7,32
3143
	stvx		v23,r3,$sp
3144
	addi		r3,r3,32
3145
	stvx		v24,r7,$sp
3146
	addi		r7,r7,32
3147
	stvx		v25,r3,$sp
3148
	addi		r3,r3,32
3149
	stvx		v26,r7,$sp
3150
	addi		r7,r7,32
3151
	stvx		v27,r3,$sp
3152
	addi		r3,r3,32
3153
	stvx		v28,r7,$sp
3154
	addi		r7,r7,32
3155
	stvx		v29,r3,$sp
3156
	addi		r3,r3,32
3157
	stvx		v30,r7,$sp
3158
	stvx		v31,r3,$sp
3159
	li		r0,-1
3160
	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
3161
	li		$x10,0x10
3162
	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
3163
	li		$x20,0x20
3164
	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
3165
	li		$x30,0x30
3166
	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
3167
	li		$x40,0x40
3168
	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
3169
	li		$x50,0x50
3170
	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
3171
	li		$x60,0x60
3172
	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
3173
	li		$x70,0x70
3174
	mtspr		256,r0
3175

3176
	xxlor		2, 32+$eighty7, 32+$eighty7
3177
	vsldoi		$eighty7,$tmp,$eighty7,1        # 0x010101..87
3178
	xxlor		1, 32+$eighty7, 32+$eighty7
3179

3180
	# Load XOR Lconsts.
3181
	mr		$x70, r6
3182
	bl		Lconsts
3183
	lxvw4x		0, $x40, r6		# load XOR contents
3184
	mr		r6, $x70
3185
	li		$x70,0x70
3186

3187
	subi		$rounds,$rounds,3	# -4 in total
3188

3189
	lvx		$rndkey0,$x00,$key1	# load key schedule
3190
	lvx		v30,$x10,$key1
3191
	addi		$key1,$key1,0x20
3192
	lvx		v31,$x00,$key1
3193
	?vperm		$rndkey0,$rndkey0,v30,$keyperm
3194
	addi		$key_,$sp,$FRAME+15
3195
	mtctr		$rounds
3196

3197
Load_xts_dec_key:
3198
	?vperm		v24,v30,v31,$keyperm
3199
	lvx		v30,$x10,$key1
3200
	addi		$key1,$key1,0x20
3201
	stvx		v24,$x00,$key_		# off-load round[1]
3202
	?vperm		v25,v31,v30,$keyperm
3203
	lvx		v31,$x00,$key1
3204
	stvx		v25,$x10,$key_		# off-load round[2]
3205
	addi		$key_,$key_,0x20
3206
	bdnz		Load_xts_dec_key
3207

3208
	lvx		v26,$x10,$key1
3209
	?vperm		v24,v30,v31,$keyperm
3210
	lvx		v27,$x20,$key1
3211
	stvx		v24,$x00,$key_		# off-load round[3]
3212
	?vperm		v25,v31,v26,$keyperm
3213
	lvx		v28,$x30,$key1
3214
	stvx		v25,$x10,$key_		# off-load round[4]
3215
	addi		$key_,$sp,$FRAME+15	# rewind $key_
3216
	?vperm		v26,v26,v27,$keyperm
3217
	lvx		v29,$x40,$key1
3218
	?vperm		v27,v27,v28,$keyperm
3219
	lvx		v30,$x50,$key1
3220
	?vperm		v28,v28,v29,$keyperm
3221
	lvx		v31,$x60,$key1
3222
	?vperm		v29,v29,v30,$keyperm
3223
	lvx		$twk5,$x70,$key1	# borrow $twk5
3224
	?vperm		v30,v30,v31,$keyperm
3225
	lvx		v24,$x00,$key_		# pre-load round[1]
3226
	?vperm		v31,v31,$twk5,$keyperm
3227
	lvx		v25,$x10,$key_		# pre-load round[2]
3228

3229
	 vperm		$in0,$inout,$inptail,$inpperm
3230
	 subi		$inp,$inp,31		# undo "caller"
3231
	vxor		$twk0,$tweak,$rndkey0
3232
	vsrab		$tmp,$tweak,$seven	# next tweak value
3233
	vaddubm		$tweak,$tweak,$tweak
3234
	vand		$tmp,$tmp,$eighty7
3235
	 vxor		$out0,$in0,$twk0
3236
	xxlor		32+$in1, 0, 0
3237
	vpermxor	$tweak, $tweak, $tmp, $in1
3238

3239
	 lvx_u		$in1,$x10,$inp
3240
	vxor		$twk1,$tweak,$rndkey0
3241
	vsrab		$tmp,$tweak,$seven	# next tweak value
3242
	vaddubm		$tweak,$tweak,$tweak
3243
	 le?vperm	$in1,$in1,$in1,$leperm
3244
	vand		$tmp,$tmp,$eighty7
3245
	 vxor		$out1,$in1,$twk1
3246
	xxlor		32+$in2, 0, 0
3247
	vpermxor	$tweak, $tweak, $tmp, $in2
3248

3249
	 lvx_u		$in2,$x20,$inp
3250
	 andi.		$taillen,$len,15
3251
	vxor		$twk2,$tweak,$rndkey0
3252
	vsrab		$tmp,$tweak,$seven	# next tweak value
3253
	vaddubm		$tweak,$tweak,$tweak
3254
	 le?vperm	$in2,$in2,$in2,$leperm
3255
	vand		$tmp,$tmp,$eighty7
3256
	 vxor		$out2,$in2,$twk2
3257
	xxlor		32+$in3, 0, 0
3258
	vpermxor	$tweak, $tweak, $tmp, $in3
3259

3260
	 lvx_u		$in3,$x30,$inp
3261
	 sub		$len,$len,$taillen
3262
	vxor		$twk3,$tweak,$rndkey0
3263
	vsrab		$tmp,$tweak,$seven	# next tweak value
3264
	vaddubm		$tweak,$tweak,$tweak
3265
	 le?vperm	$in3,$in3,$in3,$leperm
3266
	vand		$tmp,$tmp,$eighty7
3267
	 vxor		$out3,$in3,$twk3
3268
	xxlor		32+$in4, 0, 0
3269
	vpermxor	$tweak, $tweak, $tmp, $in4
3270

3271
	 lvx_u		$in4,$x40,$inp
3272
	 subi		$len,$len,0x60
3273
	vxor		$twk4,$tweak,$rndkey0
3274
	vsrab		$tmp,$tweak,$seven	# next tweak value
3275
	vaddubm		$tweak,$tweak,$tweak
3276
	 le?vperm	$in4,$in4,$in4,$leperm
3277
	vand		$tmp,$tmp,$eighty7
3278
	 vxor		$out4,$in4,$twk4
3279
	xxlor		32+$in5, 0, 0
3280
	vpermxor	$tweak, $tweak, $tmp, $in5
3281

3282
	 lvx_u		$in5,$x50,$inp
3283
	 addi		$inp,$inp,0x60
3284
	vxor		$twk5,$tweak,$rndkey0
3285
	vsrab		$tmp,$tweak,$seven	# next tweak value
3286
	vaddubm		$tweak,$tweak,$tweak
3287
	 le?vperm	$in5,$in5,$in5,$leperm
3288
	vand		$tmp,$tmp,$eighty7
3289
	 vxor		$out5,$in5,$twk5
3290
	xxlor		32+$in0, 0, 0
3291
	vpermxor	$tweak, $tweak, $tmp, $in0
3292

3293
	vxor		v31,v31,$rndkey0
3294
	mtctr		$rounds
3295
	b		Loop_xts_dec6x
3296

3297
.align	5
3298
Loop_xts_dec6x:
3299
	vncipher	$out0,$out0,v24
3300
	vncipher	$out1,$out1,v24
3301
	vncipher	$out2,$out2,v24
3302
	vncipher	$out3,$out3,v24
3303
	vncipher	$out4,$out4,v24
3304
	vncipher	$out5,$out5,v24
3305
	lvx		v24,$x20,$key_		# round[3]
3306
	addi		$key_,$key_,0x20
3307

3308
	vncipher	$out0,$out0,v25
3309
	vncipher	$out1,$out1,v25
3310
	vncipher	$out2,$out2,v25
3311
	vncipher	$out3,$out3,v25
3312
	vncipher	$out4,$out4,v25
3313
	vncipher	$out5,$out5,v25
3314
	lvx		v25,$x10,$key_		# round[4]
3315
	bdnz		Loop_xts_dec6x
3316

3317
	xxlor		32+$eighty7, 1, 1	# 0x010101..87
3318

3319
	subic		$len,$len,96		# $len-=96
3320
	 vxor		$in0,$twk0,v31		# xor with last round key
3321
	vncipher	$out0,$out0,v24
3322
	vncipher	$out1,$out1,v24
3323
	 vsrab		$tmp,$tweak,$seven	# next tweak value
3324
	 vxor		$twk0,$tweak,$rndkey0
3325
	 vaddubm	$tweak,$tweak,$tweak
3326
	vncipher	$out2,$out2,v24
3327
	vncipher	$out3,$out3,v24
3328
	vncipher	$out4,$out4,v24
3329
	vncipher	$out5,$out5,v24
3330

3331
	subfe.		r0,r0,r0		# borrow?-1:0
3332
	 vand		$tmp,$tmp,$eighty7
3333
	vncipher	$out0,$out0,v25
3334
	vncipher	$out1,$out1,v25
3335
	 xxlor		32+$in1, 0, 0
3336
	 vpermxor	$tweak, $tweak, $tmp, $in1
3337
	vncipher	$out2,$out2,v25
3338
	vncipher	$out3,$out3,v25
3339
	 vxor		$in1,$twk1,v31
3340
	 vsrab		$tmp,$tweak,$seven	# next tweak value
3341
	 vxor		$twk1,$tweak,$rndkey0
3342
	vncipher	$out4,$out4,v25
3343
	vncipher	$out5,$out5,v25
3344

3345
	and		r0,r0,$len
3346
	 vaddubm	$tweak,$tweak,$tweak
3347
	vncipher	$out0,$out0,v26
3348
	vncipher	$out1,$out1,v26
3349
	 vand		$tmp,$tmp,$eighty7
3350
	vncipher	$out2,$out2,v26
3351
	vncipher	$out3,$out3,v26
3352
	 xxlor		32+$in2, 0, 0
3353
	 vpermxor	$tweak, $tweak, $tmp, $in2
3354
	vncipher	$out4,$out4,v26
3355
	vncipher	$out5,$out5,v26
3356

3357
	add		$inp,$inp,r0		# $inp is adjusted in such
3358
						# way that at exit from the
3359
						# loop inX-in5 are loaded
3360
						# with last "words"
3361
	 vxor		$in2,$twk2,v31
3362
	 vsrab		$tmp,$tweak,$seven	# next tweak value
3363
	 vxor		$twk2,$tweak,$rndkey0
3364
	 vaddubm	$tweak,$tweak,$tweak
3365
	vncipher	$out0,$out0,v27
3366
	vncipher	$out1,$out1,v27
3367
	vncipher	$out2,$out2,v27
3368
	vncipher	$out3,$out3,v27
3369
	 vand		$tmp,$tmp,$eighty7
3370
	vncipher	$out4,$out4,v27
3371
	vncipher	$out5,$out5,v27
3372

3373
	addi		$key_,$sp,$FRAME+15	# rewind $key_
3374
	 xxlor		32+$in3, 0, 0
3375
	 vpermxor	$tweak, $tweak, $tmp, $in3
3376
	vncipher	$out0,$out0,v28
3377
	vncipher	$out1,$out1,v28
3378
	 vxor		$in3,$twk3,v31
3379
	 vsrab		$tmp,$tweak,$seven	# next tweak value
3380
	 vxor		$twk3,$tweak,$rndkey0
3381
	vncipher	$out2,$out2,v28
3382
	vncipher	$out3,$out3,v28
3383
	 vaddubm	$tweak,$tweak,$tweak
3384
	vncipher	$out4,$out4,v28
3385
	vncipher	$out5,$out5,v28
3386
	lvx		v24,$x00,$key_		# re-pre-load round[1]
3387
	 vand		$tmp,$tmp,$eighty7
3388

3389
	vncipher	$out0,$out0,v29
3390
	vncipher	$out1,$out1,v29
3391
	 xxlor		32+$in4, 0, 0
3392
	 vpermxor	$tweak, $tweak, $tmp, $in4
3393
	vncipher	$out2,$out2,v29
3394
	vncipher	$out3,$out3,v29
3395
	 vxor		$in4,$twk4,v31
3396
	 vsrab		$tmp,$tweak,$seven	# next tweak value
3397
	 vxor		$twk4,$tweak,$rndkey0
3398
	vncipher	$out4,$out4,v29
3399
	vncipher	$out5,$out5,v29
3400
	lvx		v25,$x10,$key_		# re-pre-load round[2]
3401
	 vaddubm	$tweak,$tweak,$tweak
3402

3403
	vncipher	$out0,$out0,v30
3404
	vncipher	$out1,$out1,v30
3405
	 vand		$tmp,$tmp,$eighty7
3406
	vncipher	$out2,$out2,v30
3407
	vncipher	$out3,$out3,v30
3408
	 xxlor		32+$in5, 0, 0
3409
	 vpermxor	$tweak, $tweak, $tmp, $in5
3410
	vncipher	$out4,$out4,v30
3411
	vncipher	$out5,$out5,v30
3412
	 vxor		$in5,$twk5,v31
3413
	 vsrab		$tmp,$tweak,$seven	# next tweak value
3414
	 vxor		$twk5,$tweak,$rndkey0
3415

3416
	vncipherlast	$out0,$out0,$in0
3417
	 lvx_u		$in0,$x00,$inp		# load next input block
3418
	 vaddubm	$tweak,$tweak,$tweak
3419
	vncipherlast	$out1,$out1,$in1
3420
	 lvx_u		$in1,$x10,$inp
3421
	vncipherlast	$out2,$out2,$in2
3422
	 le?vperm	$in0,$in0,$in0,$leperm
3423
	 lvx_u		$in2,$x20,$inp
3424
	 vand		$tmp,$tmp,$eighty7
3425
	vncipherlast	$out3,$out3,$in3
3426
	 le?vperm	$in1,$in1,$in1,$leperm
3427
	 lvx_u		$in3,$x30,$inp
3428
	vncipherlast	$out4,$out4,$in4
3429
	 le?vperm	$in2,$in2,$in2,$leperm
3430
	 lvx_u		$in4,$x40,$inp
3431
	 xxlor		10, 32+$in0, 32+$in0
3432
	 xxlor		32+$in0, 0, 0
3433
	 vpermxor	$tweak, $tweak, $tmp, $in0
3434
	 xxlor		32+$in0, 10, 10
3435
	vncipherlast	$out5,$out5,$in5
3436
	 le?vperm	$in3,$in3,$in3,$leperm
3437
	 lvx_u		$in5,$x50,$inp
3438
	 addi		$inp,$inp,0x60
3439
	 le?vperm	$in4,$in4,$in4,$leperm
3440
	 le?vperm	$in5,$in5,$in5,$leperm
3441

3442
	le?vperm	$out0,$out0,$out0,$leperm
3443
	le?vperm	$out1,$out1,$out1,$leperm
3444
	stvx_u		$out0,$x00,$out		# store output
3445
	 vxor		$out0,$in0,$twk0
3446
	le?vperm	$out2,$out2,$out2,$leperm
3447
	stvx_u		$out1,$x10,$out
3448
	 vxor		$out1,$in1,$twk1
3449
	le?vperm	$out3,$out3,$out3,$leperm
3450
	stvx_u		$out2,$x20,$out
3451
	 vxor		$out2,$in2,$twk2
3452
	le?vperm	$out4,$out4,$out4,$leperm
3453
	stvx_u		$out3,$x30,$out
3454
	 vxor		$out3,$in3,$twk3
3455
	le?vperm	$out5,$out5,$out5,$leperm
3456
	stvx_u		$out4,$x40,$out
3457
	 vxor		$out4,$in4,$twk4
3458
	stvx_u		$out5,$x50,$out
3459
	 vxor		$out5,$in5,$twk5
3460
	addi		$out,$out,0x60
3461

3462
	mtctr		$rounds
3463
	beq		Loop_xts_dec6x		# did $len-=96 borrow?
3464

3465
	xxlor		32+$eighty7, 2, 2	# 0x010101..87
3466

3467
	addic.		$len,$len,0x60
3468
	beq		Lxts_dec6x_zero
3469
	cmpwi		$len,0x20
3470
	blt		Lxts_dec6x_one
3471
	nop
3472
	beq		Lxts_dec6x_two
3473
	cmpwi		$len,0x40
3474
	blt		Lxts_dec6x_three
3475
	nop
3476
	beq		Lxts_dec6x_four
3477

3478
Lxts_dec6x_five:
3479
	vxor		$out0,$in1,$twk0
3480
	vxor		$out1,$in2,$twk1
3481
	vxor		$out2,$in3,$twk2
3482
	vxor		$out3,$in4,$twk3
3483
	vxor		$out4,$in5,$twk4
3484

3485
	bl		_aesp8_xts_dec5x
3486

3487
	le?vperm	$out0,$out0,$out0,$leperm
3488
	vmr		$twk0,$twk5		# unused tweak
3489
	vxor		$twk1,$tweak,$rndkey0
3490
	le?vperm	$out1,$out1,$out1,$leperm
3491
	stvx_u		$out0,$x00,$out		# store output
3492
	vxor		$out0,$in0,$twk1
3493
	le?vperm	$out2,$out2,$out2,$leperm
3494
	stvx_u		$out1,$x10,$out
3495
	le?vperm	$out3,$out3,$out3,$leperm
3496
	stvx_u		$out2,$x20,$out
3497
	le?vperm	$out4,$out4,$out4,$leperm
3498
	stvx_u		$out3,$x30,$out
3499
	stvx_u		$out4,$x40,$out
3500
	addi		$out,$out,0x50
3501
	bne		Lxts_dec6x_steal
3502
	b		Lxts_dec6x_done
3503

3504
.align	4
3505
Lxts_dec6x_four:
3506
	vxor		$out0,$in2,$twk0
3507
	vxor		$out1,$in3,$twk1
3508
	vxor		$out2,$in4,$twk2
3509
	vxor		$out3,$in5,$twk3
3510
	vxor		$out4,$out4,$out4
3511

3512
	bl		_aesp8_xts_dec5x
3513

3514
	le?vperm	$out0,$out0,$out0,$leperm
3515
	vmr		$twk0,$twk4		# unused tweak
3516
	vmr		$twk1,$twk5
3517
	le?vperm	$out1,$out1,$out1,$leperm
3518
	stvx_u		$out0,$x00,$out		# store output
3519
	vxor		$out0,$in0,$twk5
3520
	le?vperm	$out2,$out2,$out2,$leperm
3521
	stvx_u		$out1,$x10,$out
3522
	le?vperm	$out3,$out3,$out3,$leperm
3523
	stvx_u		$out2,$x20,$out
3524
	stvx_u		$out3,$x30,$out
3525
	addi		$out,$out,0x40
3526
	bne		Lxts_dec6x_steal
3527
	b		Lxts_dec6x_done
3528

3529
.align	4
3530
Lxts_dec6x_three:
3531
	vxor		$out0,$in3,$twk0
3532
	vxor		$out1,$in4,$twk1
3533
	vxor		$out2,$in5,$twk2
3534
	vxor		$out3,$out3,$out3
3535
	vxor		$out4,$out4,$out4
3536

3537
	bl		_aesp8_xts_dec5x
3538

3539
	le?vperm	$out0,$out0,$out0,$leperm
3540
	vmr		$twk0,$twk3		# unused tweak
3541
	vmr		$twk1,$twk4
3542
	le?vperm	$out1,$out1,$out1,$leperm
3543
	stvx_u		$out0,$x00,$out		# store output
3544
	vxor		$out0,$in0,$twk4
3545
	le?vperm	$out2,$out2,$out2,$leperm
3546
	stvx_u		$out1,$x10,$out
3547
	stvx_u		$out2,$x20,$out
3548
	addi		$out,$out,0x30
3549
	bne		Lxts_dec6x_steal
3550
	b		Lxts_dec6x_done
3551

3552
.align	4
3553
Lxts_dec6x_two:
3554
	vxor		$out0,$in4,$twk0
3555
	vxor		$out1,$in5,$twk1
3556
	vxor		$out2,$out2,$out2
3557
	vxor		$out3,$out3,$out3
3558
	vxor		$out4,$out4,$out4
3559

3560
	bl		_aesp8_xts_dec5x
3561

3562
	le?vperm	$out0,$out0,$out0,$leperm
3563
	vmr		$twk0,$twk2		# unused tweak
3564
	vmr		$twk1,$twk3
3565
	le?vperm	$out1,$out1,$out1,$leperm
3566
	stvx_u		$out0,$x00,$out		# store output
3567
	vxor		$out0,$in0,$twk3
3568
	stvx_u		$out1,$x10,$out
3569
	addi		$out,$out,0x20
3570
	bne		Lxts_dec6x_steal
3571
	b		Lxts_dec6x_done
3572

3573
.align	4
3574
Lxts_dec6x_one:
3575
	vxor		$out0,$in5,$twk0
3576
	nop
3577
Loop_xts_dec1x:
3578
	vncipher	$out0,$out0,v24
3579
	lvx		v24,$x20,$key_		# round[3]
3580
	addi		$key_,$key_,0x20
3581

3582
	vncipher	$out0,$out0,v25
3583
	lvx		v25,$x10,$key_		# round[4]
3584
	bdnz		Loop_xts_dec1x
3585

3586
	subi		r0,$taillen,1
3587
	vncipher	$out0,$out0,v24
3588

3589
	andi.		r0,r0,16
3590
	cmpwi		$taillen,0
3591
	vncipher	$out0,$out0,v25
3592

3593
	sub		$inp,$inp,r0
3594
	vncipher	$out0,$out0,v26
3595

3596
	lvx_u		$in0,0,$inp
3597
	vncipher	$out0,$out0,v27
3598

3599
	addi		$key_,$sp,$FRAME+15	# rewind $key_
3600
	vncipher	$out0,$out0,v28
3601
	lvx		v24,$x00,$key_		# re-pre-load round[1]
3602

3603
	vncipher	$out0,$out0,v29
3604
	lvx		v25,$x10,$key_		# re-pre-load round[2]
3605
	 vxor		$twk0,$twk0,v31
3606

3607
	le?vperm	$in0,$in0,$in0,$leperm
3608
	vncipher	$out0,$out0,v30
3609

3610
	mtctr		$rounds
3611
	vncipherlast	$out0,$out0,$twk0
3612

3613
	vmr		$twk0,$twk1		# unused tweak
3614
	vmr		$twk1,$twk2
3615
	le?vperm	$out0,$out0,$out0,$leperm
3616
	stvx_u		$out0,$x00,$out		# store output
3617
	addi		$out,$out,0x10
3618
	vxor		$out0,$in0,$twk2
3619
	bne		Lxts_dec6x_steal
3620
	b		Lxts_dec6x_done
3621

3622
.align	4
3623
Lxts_dec6x_zero:
3624
	cmpwi		$taillen,0
3625
	beq		Lxts_dec6x_done
3626

3627
	lvx_u		$in0,0,$inp
3628
	le?vperm	$in0,$in0,$in0,$leperm
3629
	vxor		$out0,$in0,$twk1
3630
Lxts_dec6x_steal:
3631
	vncipher	$out0,$out0,v24
3632
	lvx		v24,$x20,$key_		# round[3]
3633
	addi		$key_,$key_,0x20
3634

3635
	vncipher	$out0,$out0,v25
3636
	lvx		v25,$x10,$key_		# round[4]
3637
	bdnz		Lxts_dec6x_steal
3638

3639
	add		$inp,$inp,$taillen
3640
	vncipher	$out0,$out0,v24
3641

3642
	cmpwi		$taillen,0
3643
	vncipher	$out0,$out0,v25
3644

3645
	lvx_u		$in0,0,$inp
3646
	vncipher	$out0,$out0,v26
3647

3648
	lvsr		$inpperm,0,$taillen	# $in5 is no more
3649
	vncipher	$out0,$out0,v27
3650

3651
	addi		$key_,$sp,$FRAME+15	# rewind $key_
3652
	vncipher	$out0,$out0,v28
3653
	lvx		v24,$x00,$key_		# re-pre-load round[1]
3654

3655
	vncipher	$out0,$out0,v29
3656
	lvx		v25,$x10,$key_		# re-pre-load round[2]
3657
	 vxor		$twk1,$twk1,v31
3658

3659
	le?vperm	$in0,$in0,$in0,$leperm
3660
	vncipher	$out0,$out0,v30
3661

3662
	vperm		$in0,$in0,$in0,$inpperm
3663
	vncipherlast	$tmp,$out0,$twk1
3664

3665
	le?vperm	$out0,$tmp,$tmp,$leperm
3666
	le?stvx_u	$out0,0,$out
3667
	be?stvx_u	$tmp,0,$out
3668

3669
	vxor		$out0,$out0,$out0
3670
	vspltisb	$out1,-1
3671
	vperm		$out0,$out0,$out1,$inpperm
3672
	vsel		$out0,$in0,$tmp,$out0
3673
	vxor		$out0,$out0,$twk0
3674

3675
	subi		r30,$out,1
3676
	mtctr		$taillen
3677
Loop_xts_dec6x_steal:
3678
	lbzu		r0,1(r30)
3679
	stb		r0,16(r30)
3680
	bdnz		Loop_xts_dec6x_steal
3681

3682
	li		$taillen,0
3683
	mtctr		$rounds
3684
	b		Loop_xts_dec1x		# one more time...
3685

3686
.align	4
3687
Lxts_dec6x_done:
3688
	${UCMP}i	$ivp,0
3689
	beq		Lxts_dec6x_ret
3690

3691
	vxor		$tweak,$twk0,$rndkey0
3692
	le?vperm	$tweak,$tweak,$tweak,$leperm
3693
	stvx_u		$tweak,0,$ivp
3694

3695
Lxts_dec6x_ret:
3696
	mtlr		r11
3697
	li		r10,`$FRAME+15`
3698
	li		r11,`$FRAME+31`
3699
	stvx		$seven,r10,$sp		# wipe copies of round keys
3700
	addi		r10,r10,32
3701
	stvx		$seven,r11,$sp
3702
	addi		r11,r11,32
3703
	stvx		$seven,r10,$sp
3704
	addi		r10,r10,32
3705
	stvx		$seven,r11,$sp
3706
	addi		r11,r11,32
3707
	stvx		$seven,r10,$sp
3708
	addi		r10,r10,32
3709
	stvx		$seven,r11,$sp
3710
	addi		r11,r11,32
3711
	stvx		$seven,r10,$sp
3712
	addi		r10,r10,32
3713
	stvx		$seven,r11,$sp
3714
	addi		r11,r11,32
3715

3716
	mtspr		256,$vrsave
3717
	lvx		v20,r10,$sp		# ABI says so
3718
	addi		r10,r10,32
3719
	lvx		v21,r11,$sp
3720
	addi		r11,r11,32
3721
	lvx		v22,r10,$sp
3722
	addi		r10,r10,32
3723
	lvx		v23,r11,$sp
3724
	addi		r11,r11,32
3725
	lvx		v24,r10,$sp
3726
	addi		r10,r10,32
3727
	lvx		v25,r11,$sp
3728
	addi		r11,r11,32
3729
	lvx		v26,r10,$sp
3730
	addi		r10,r10,32
3731
	lvx		v27,r11,$sp
3732
	addi		r11,r11,32
3733
	lvx		v28,r10,$sp
3734
	addi		r10,r10,32
3735
	lvx		v29,r11,$sp
3736
	addi		r11,r11,32
3737
	lvx		v30,r10,$sp
3738
	lvx		v31,r11,$sp
3739
	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
3740
	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
3741
	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
3742
	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
3743
	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
3744
	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
3745
	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
3746
	blr
3747
	.long		0
3748
	.byte		0,12,0x04,1,0x80,6,6,0
3749
	.long		0
3750

3751
.align	5
3752
_aesp8_xts_dec5x:
3753
	vncipher	$out0,$out0,v24
3754
	vncipher	$out1,$out1,v24
3755
	vncipher	$out2,$out2,v24
3756
	vncipher	$out3,$out3,v24
3757
	vncipher	$out4,$out4,v24
3758
	lvx		v24,$x20,$key_		# round[3]
3759
	addi		$key_,$key_,0x20
3760

3761
	vncipher	$out0,$out0,v25
3762
	vncipher	$out1,$out1,v25
3763
	vncipher	$out2,$out2,v25
3764
	vncipher	$out3,$out3,v25
3765
	vncipher	$out4,$out4,v25
3766
	lvx		v25,$x10,$key_		# round[4]
3767
	bdnz		_aesp8_xts_dec5x
3768

3769
	subi		r0,$taillen,1
3770
	vncipher	$out0,$out0,v24
3771
	vncipher	$out1,$out1,v24
3772
	vncipher	$out2,$out2,v24
3773
	vncipher	$out3,$out3,v24
3774
	vncipher	$out4,$out4,v24
3775

3776
	andi.		r0,r0,16
3777
	cmpwi		$taillen,0
3778
	vncipher	$out0,$out0,v25
3779
	vncipher	$out1,$out1,v25
3780
	vncipher	$out2,$out2,v25
3781
	vncipher	$out3,$out3,v25
3782
	vncipher	$out4,$out4,v25
3783
	 vxor		$twk0,$twk0,v31
3784

3785
	sub		$inp,$inp,r0
3786
	vncipher	$out0,$out0,v26
3787
	vncipher	$out1,$out1,v26
3788
	vncipher	$out2,$out2,v26
3789
	vncipher	$out3,$out3,v26
3790
	vncipher	$out4,$out4,v26
3791
	 vxor		$in1,$twk1,v31
3792

3793
	vncipher	$out0,$out0,v27
3794
	lvx_u		$in0,0,$inp
3795
	vncipher	$out1,$out1,v27
3796
	vncipher	$out2,$out2,v27
3797
	vncipher	$out3,$out3,v27
3798
	vncipher	$out4,$out4,v27
3799
	 vxor		$in2,$twk2,v31
3800

3801
	addi		$key_,$sp,$FRAME+15	# rewind $key_
3802
	vncipher	$out0,$out0,v28
3803
	vncipher	$out1,$out1,v28
3804
	vncipher	$out2,$out2,v28
3805
	vncipher	$out3,$out3,v28
3806
	vncipher	$out4,$out4,v28
3807
	lvx		v24,$x00,$key_		# re-pre-load round[1]
3808
	 vxor		$in3,$twk3,v31
3809

3810
	vncipher	$out0,$out0,v29
3811
	le?vperm	$in0,$in0,$in0,$leperm
3812
	vncipher	$out1,$out1,v29
3813
	vncipher	$out2,$out2,v29
3814
	vncipher	$out3,$out3,v29
3815
	vncipher	$out4,$out4,v29
3816
	lvx		v25,$x10,$key_		# re-pre-load round[2]
3817
	 vxor		$in4,$twk4,v31
3818

3819
	vncipher	$out0,$out0,v30
3820
	vncipher	$out1,$out1,v30
3821
	vncipher	$out2,$out2,v30
3822
	vncipher	$out3,$out3,v30
3823
	vncipher	$out4,$out4,v30
3824

3825
	vncipherlast	$out0,$out0,$twk0
3826
	vncipherlast	$out1,$out1,$in1
3827
	vncipherlast	$out2,$out2,$in2
3828
	vncipherlast	$out3,$out3,$in3
3829
	vncipherlast	$out4,$out4,$in4
3830
	mtctr		$rounds
3831
	blr
3832
        .long   	0
3833
        .byte   	0,12,0x14,0,0,0,0,0
3834
___
3835
}}	}}}
3836

3837
my $consts=1;
3838
foreach(split("\n",$code)) {
3839
        s/\`([^\`]*)\`/eval($1)/geo;
3840

3841
	# constants table endian-specific conversion
3842
	if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
3843
	    my $conv=$3;
3844
	    my @bytes=();
3845

3846
	    # convert to endian-agnostic format
3847
	    if ($1 eq "long") {
3848
	      foreach (split(/,\s*/,$2)) {
3849
		my $l = /^0/?oct:int;
3850
		push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff;
3851
	      }
3852
	    } else {
3853
		@bytes = map(/^0/?oct:int,split(/,\s*/,$2));
3854
	    }
3855

3856
	    # little-endian conversion
3857
	    if ($flavour =~ /le$/o) {
3858
		SWITCH: for($conv)  {
3859
		    /\?inv/ && do   { @bytes=map($_^0xf,@bytes); last; };
3860
		    /\?rev/ && do   { @bytes=reverse(@bytes);    last; };
3861
		}
3862
	    }
3863

3864
	    #emit
3865
	    print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n";
3866
	    next;
3867
	}
3868
	$consts=0 if (m/Lconsts:/o);	# end of table
3869

3870
	# instructions prefixed with '?' are endian-specific and need
3871
	# to be adjusted accordingly...
3872
	if ($flavour =~ /le$/o) {	# little-endian
3873
	    s/le\?//o		or
3874
	    s/be\?/#be#/o	or
3875
	    s/\?lvsr/lvsl/o	or
3876
	    s/\?lvsl/lvsr/o	or
3877
	    s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
3878
	    s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
3879
	    s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
3880
	} else {			# big-endian
3881
	    s/le\?/#le#/o	or
3882
	    s/be\?//o		or
3883
	    s/\?([a-z]+)/$1/o;
3884
	}
3885

3886
        print $_,"\n";
3887
}
3888

3889
close STDOUT;
3890

3891
Product

Resources

Company
