aes-gcm-p10.S

/* SPDX-License-Identifier: GPL-2.0-or-later */
#
# Accelerated AES-GCM stitched implementation for ppc64le.
#
# Copyright 2024- IBM Inc.
#
#===================================================================================
# Written by Danny Tsen <[email protected]>
#
# GHASH is based on the Karatsuba multiplication method.
#
#    Xi xor X1
#
#    X1 * H^4 + X2 * H^3 + x3 * H^2 + X4 * H =
#      (X1.h * H4.h + xX.l * H4.l + X1 * H4) +
#      (X2.h * H3.h + X2.l * H3.l + X2 * H3) +
#      (X3.h * H2.h + X3.l * H2.l + X3 * H2) +
#      (X4.h * H.h + X4.l * H.l + X4 * H)
#
# Xi = v0
# H Poly = v2
# Hash keys = v3 - v14
#     ( H.l, H, H.h)
#     ( H^2.l, H^2, H^2.h)
#     ( H^3.l, H^3, H^3.h)
#     ( H^4.l, H^4, H^4.h)
#
# v30 is IV
# v31 - counter 1
#
# AES used,
#     vs0 - round key 0
#     v15, v16, v17, v18, v19, v20, v21, v22 for 8 blocks (encrypted)
#
# This implementation uses stitched AES-GCM approach to improve overall performance.
# AES is implemented with 8x blocks and GHASH is using 2 4x blocks.
#
# ===================================================================================
#

#include <asm/ppc_asm.h>
#include <linux/linkage.h>

.machine        "any"
.text

.macro	SAVE_GPR GPR OFFSET FRAME
	std	\GPR,\OFFSET(\FRAME)
.endm

.macro	SAVE_VRS VRS OFFSET FRAME
	stxv	\VRS+32, \OFFSET(\FRAME)
.endm

.macro	RESTORE_GPR GPR OFFSET FRAME
	ld	\GPR,\OFFSET(\FRAME)
.endm

.macro	RESTORE_VRS VRS OFFSET FRAME
	lxv	\VRS+32, \OFFSET(\FRAME)
.endm

.macro SAVE_REGS
	mflr 0
	std 0, 16(1)
	stdu 1,-512(1)

	SAVE_GPR 14, 112, 1
	SAVE_GPR 15, 120, 1
	SAVE_GPR 16, 128, 1
	SAVE_GPR 17, 136, 1
	SAVE_GPR 18, 144, 1
	SAVE_GPR 19, 152, 1
	SAVE_GPR 20, 160, 1
	SAVE_GPR 21, 168, 1
	SAVE_GPR 22, 176, 1
	SAVE_GPR 23, 184, 1
	SAVE_GPR 24, 192, 1

	addi	9, 1, 256
	SAVE_VRS 20, 0, 9
	SAVE_VRS 21, 16, 9
	SAVE_VRS 22, 32, 9
	SAVE_VRS 23, 48, 9
	SAVE_VRS 24, 64, 9
	SAVE_VRS 25, 80, 9
	SAVE_VRS 26, 96, 9
	SAVE_VRS 27, 112, 9
	SAVE_VRS 28, 128, 9
	SAVE_VRS 29, 144, 9
	SAVE_VRS 30, 160, 9
	SAVE_VRS 31, 176, 9
.endm # SAVE_REGS

.macro RESTORE_REGS
	addi	9, 1, 256
	RESTORE_VRS 20, 0, 9
	RESTORE_VRS 21, 16, 9
	RESTORE_VRS 22, 32, 9
	RESTORE_VRS 23, 48, 9
	RESTORE_VRS 24, 64, 9
	RESTORE_VRS 25, 80, 9
	RESTORE_VRS 26, 96, 9
	RESTORE_VRS 27, 112, 9
	RESTORE_VRS 28, 128, 9
	RESTORE_VRS 29, 144, 9
	RESTORE_VRS 30, 160, 9
	RESTORE_VRS 31, 176, 9

	RESTORE_GPR 14, 112, 1
	RESTORE_GPR 15, 120, 1
	RESTORE_GPR 16, 128, 1
	RESTORE_GPR 17, 136, 1
	RESTORE_GPR 18, 144, 1
	RESTORE_GPR 19, 152, 1
	RESTORE_GPR 20, 160, 1
	RESTORE_GPR 21, 168, 1
	RESTORE_GPR 22, 176, 1
	RESTORE_GPR 23, 184, 1
	RESTORE_GPR 24, 192, 1

	addi    1, 1, 512
	ld 0, 16(1)
	mtlr 0
.endm # RESTORE_REGS

# 4x loops
.macro AES_CIPHER_4x _VCIPHER ST r
	\_VCIPHER	\ST, \ST, \r
	\_VCIPHER	\ST+1, \ST+1, \r
	\_VCIPHER	\ST+2, \ST+2, \r
	\_VCIPHER	\ST+3, \ST+3, \r
.endm

# 8x loops
.macro AES_CIPHER_8x _VCIPHER ST r
	\_VCIPHER	\ST, \ST, \r
	\_VCIPHER	\ST+1, \ST+1, \r
	\_VCIPHER	\ST+2, \ST+2, \r
	\_VCIPHER	\ST+3, \ST+3, \r
	\_VCIPHER	\ST+4, \ST+4, \r
	\_VCIPHER	\ST+5, \ST+5, \r
	\_VCIPHER	\ST+6, \ST+6, \r
	\_VCIPHER	\ST+7, \ST+7, \r
.endm

.macro LOOP_8AES_STATE
	xxlor	32+23, 1, 1
	xxlor	32+24, 2, 2
	xxlor	32+25, 3, 3
	xxlor	32+26, 4, 4
	AES_CIPHER_8x vcipher, 15, 23
	AES_CIPHER_8x vcipher, 15, 24
	AES_CIPHER_8x vcipher, 15, 25
	AES_CIPHER_8x vcipher, 15, 26
	xxlor	32+23, 5, 5
	xxlor	32+24, 6, 6
	xxlor	32+25, 7, 7
	xxlor	32+26, 8, 8
	AES_CIPHER_8x vcipher, 15, 23
	AES_CIPHER_8x vcipher, 15, 24
	AES_CIPHER_8x vcipher, 15, 25
	AES_CIPHER_8x vcipher, 15, 26
.endm

#
# PPC_GHASH4x(H, S1, S2, S3, S4): Compute 4x hash values based on Karatsuba method.
# H: returning digest
# S#: states
#
# S1 should xor with the previous digest
#
# Xi = v0
# H Poly = v2
# Hash keys = v3 - v14
# Scratch: v23 - v29
#
.macro PPC_GHASH4x H S1 S2 S3 S4

	vpmsumd	23, 12, \S1		# H4.L * X.L
	vpmsumd	24, 9, \S2
	vpmsumd	25, 6, \S3
	vpmsumd	26, 3, \S4

	vpmsumd	27, 13, \S1		# H4.L * X.H + H4.H * X.L
	vpmsumd	28, 10, \S2		# H3.L * X1.H + H3.H * X1.L

	vxor	23, 23, 24
	vxor	23, 23, 25
	vxor	23, 23, 26		# L

	vxor	24, 27, 28
	vpmsumd	25, 7, \S3
	vpmsumd	26, 4, \S4

	vxor	24, 24, 25
	vxor	24, 24, 26		# M

	# sum hash and reduction with H Poly
	vpmsumd	28, 23, 2		# reduction

	vxor	1, 1, 1
	vsldoi	25, 24, 1, 8		# mL
	vsldoi	1, 1, 24, 8		# mH
	vxor	23, 23, 25		# mL + L

	# This performs swap and xor like,
	#   vsldoi	23, 23, 23, 8		# swap
	#   vxor	23, 23, 28
	xxlor	32+25, 10, 10
	vpermxor 23, 23, 28, 25

	vpmsumd	26, 14, \S1		# H4.H * X.H
	vpmsumd	27, 11, \S2
	vpmsumd	28, 8, \S3
	vpmsumd	29, 5, \S4

	vxor	24, 26, 27
	vxor	24, 24, 28
	vxor	24, 24, 29

	vxor	24, 24, 1

	# sum hash and reduction with H Poly
	vsldoi	25, 23, 23, 8		# swap
	vpmsumd	23, 23, 2
	vxor	27, 25, 24
	vxor	\H, 23, 27
.endm

#
# Compute update single ghash
# scratch: v1, v22..v27
#
.macro PPC_GHASH1x H S1

	vxor	1, 1, 1

	vpmsumd	22, 3, \S1		# L
	vpmsumd	23, 4, \S1		# M
	vpmsumd	24, 5, \S1		# H

	vpmsumd	27, 22, 2		# reduction

	vsldoi	25, 23, 1, 8		# mL
	vsldoi	26, 1, 23, 8		# mH
	vxor	22, 22, 25		# LL + LL
	vxor	24, 24, 26		# HH + HH

	xxlor	32+25, 10, 10
	vpermxor 22, 22, 27, 25

	vsldoi	23, 22, 22, 8		# swap
	vpmsumd	22, 22, 2		# reduction
	vxor	23, 23, 24
	vxor	\H, 22, 23
.endm

#
# LOAD_HASH_TABLE
# Xi = v0
# H Poly = v2
# Hash keys = v3 - v14
#
.macro LOAD_HASH_TABLE
	# Load Xi
	lxvb16x	32, 0, 8	# load Xi

	# load Hash - h^4, h^3, h^2, h
	li	10, 32
	lxvd2x	2+32, 10, 8	# H Poli
	li	10, 48
	lxvd2x	3+32, 10, 8	# Hl
	li	10, 64
	lxvd2x	4+32, 10, 8	# H
	li	10, 80
	lxvd2x	5+32, 10, 8	# Hh

	li	10, 96
	lxvd2x	6+32, 10, 8	# H^2l
	li	10, 112
	lxvd2x	7+32, 10, 8	# H^2
	li	10, 128
	lxvd2x	8+32, 10, 8	# H^2h

	li	10, 144
	lxvd2x	9+32, 10, 8	# H^3l
	li	10, 160
	lxvd2x	10+32, 10, 8	# H^3
	li	10, 176
	lxvd2x	11+32, 10, 8	# H^3h

	li	10, 192
	lxvd2x	12+32, 10, 8	# H^4l
	li	10, 208
	lxvd2x	13+32, 10, 8	# H^4
	li	10, 224
	lxvd2x	14+32, 10, 8	# H^4h
.endm

################################################################################
# Compute AES and ghash one block at a time.
# r23: AES rounds
# v30: current IV
# vs0: roundkey 0
#
################################################################################
SYM_FUNC_START_LOCAL(aes_gcm_crypt_1x)

	cmpdi	5, 16
	bge	__More_1x
	blr
__More_1x:
	li      10, 16
	divdu   12, 5, 10

	xxlxor	32+15, 32+30, 0

	# Pre-load 8 AES rounds to scratch vectors.
	xxlor	32+16, 1, 1
	xxlor	32+17, 2, 2
	xxlor	32+18, 3, 3
	xxlor	32+19, 4, 4
	xxlor	32+20, 5, 5
	xxlor	32+21, 6, 6
	xxlor	32+28, 7, 7
	xxlor	32+29, 8, 8
	lwz	23, 240(6)	# n rounds
	addi	22, 23, -9	# remaing AES rounds

	cmpdi	12, 0
	bgt	__Loop_1x
	blr

__Loop_1x:
	mtctr	22
	addi	10, 6, 144
	vcipher	15, 15, 16
	vcipher	15, 15, 17
	vcipher	15, 15, 18
	vcipher	15, 15, 19
	vcipher	15, 15, 20
	vcipher	15, 15, 21
	vcipher	15, 15, 28
	vcipher	15, 15, 29

__Loop_aes_1state:
	lxv	32+1, 0(10)
	vcipher	15, 15, 1
	addi	10, 10, 16
	bdnz	__Loop_aes_1state
	lxv	32+1, 0(10)		# last round key
	lxvb16x 11, 0, 14		# load input block
	vcipherlast 15, 15, 1

	xxlxor	32+15, 32+15, 11
	stxvb16x 32+15, 0, 9	# store output
	addi	14, 14, 16
	addi	9, 9, 16

	cmpdi	24, 0	# decrypt?
	bne	__Encrypt_1x
	xxlor	15+32, 11, 11
__Encrypt_1x:
	vxor	15, 15, 0
	PPC_GHASH1x 0, 15

	addi	5, 5, -16
	addi	11, 11, 16

	vadduwm 30, 30, 31		# IV + counter
	xxlxor	32+15, 32+30, 0
	addi	12, 12, -1
	cmpdi	12, 0
	bgt	__Loop_1x

	stxvb16x 32+30, 0, 7		# update IV
	stxvb16x 32+0, 0, 8		# update Xi
	blr
SYM_FUNC_END(aes_gcm_crypt_1x)

################################################################################
# Process a normal partial block when we come here.
#  Compute partial mask, Load and store partial block to stack.
#  Update partial_len and pblock.
#  pblock is (encrypted ^ AES state) for encrypt
#        and (input ^ AES state) for decrypt.
#
################################################################################
SYM_FUNC_START_LOCAL(__Process_partial)

	# create partial mask
	vspltisb 16, -1
	li	12, 16
	sub	12, 12, 5
	sldi	12, 12, 3
	mtvsrdd	32+17, 0, 12
	vslo	16, 16, 17		# partial block mask

	lxvb16x 11, 0, 14		# load partial block
	xxland	11, 11, 32+16

	# AES crypt partial
	xxlxor	32+15, 32+30, 0
	lwz	23, 240(6)		# n rounds
	addi	22, 23, -1		# loop - 1
	mtctr	22
	addi	10, 6, 16

__Loop_aes_pstate:
	lxv	32+1, 0(10)
	vcipher	15, 15, 1
	addi	10, 10, 16
	bdnz	__Loop_aes_pstate
	lxv	32+1, 0(10)		# last round key
	vcipherlast 15, 15, 1

	xxlxor	32+15, 32+15, 11
	vand	15, 15, 16

	# AES crypt output v15
	# Write partial
	li	10, 224
	stxvb16x 15+32, 10, 1		# write v15 to stack
	addi	10, 1, 223
	addi	12, 9, -1
        mtctr	5			# partial block len
__Write_partial:
        lbzu	22, 1(10)
	stbu	22, 1(12)
        bdnz	__Write_partial

	cmpdi	24, 0			# decrypt?
	bne	__Encrypt_partial
	xxlor	32+15, 11, 11		# decrypt using the input block
__Encrypt_partial:
	#vxor	15, 15, 0		# ^ previous hash
	#PPC_GHASH1x 0, 15

	add	14, 14, 5
	add	9, 9, 5
	std	5, 56(7)		# update partial
	sub	11, 11, 5
	li	5, 0			# done last byte

	#
	# Don't increase IV since this is the last partial.
	# It should get updated in gcm_update if no more data blocks.
	#vadduwm	30, 30, 31		# increase IV
	stxvb16x 32+30, 0, 7		# update IV
	li	10, 64
	stxvb16x 32+0, 0, 8		# Update X1
	stxvb16x 32+15, 10, 7		# Update pblock
	blr
SYM_FUNC_END(__Process_partial)

################################################################################
# Combine partial blocks and ghash when we come here.
#
# The partial block has to be shifted to the right location to encrypt/decrypt
# and compute ghash if combing the previous partial block is needed.
# - Compute ghash for a full block. Clear Partial_len and pblock. Update IV.
#   Write Xi.
# - Don't compute ghash if not full block.  gcm_update will take care of it
#   is the last block. Update Partial_len and pblock.
#
################################################################################
SYM_FUNC_START_LOCAL(__Combine_partial)

	ld	12, 56(7)
	mr	21, 5			# these bytes to be processed

	li	17, 0
	li	16, 16
	sub	22, 16, 12		# bytes to complete a block
	sub	17, 22, 5		# remaining bytes in a block
	cmpdi	5, 16
	ble	__Inp_msg_less16
	li	17, 0
	mr	21, 22
	b	__Combine_continue
__Inp_msg_less16:
	cmpd	22, 5
	bgt	__Combine_continue
	li	17, 0
	mr	21, 22			# these bytes to be processed

__Combine_continue:
	# load msg and shift to the proper location and mask
	vspltisb 16, -1
	sldi	15, 12, 3
	mtvsrdd	32+17, 0, 15
	vslo	16, 16, 17
	vsro	16, 16, 17
	sldi	15, 17, 3
	mtvsrdd	32+17, 0, 15
	vsro	16, 16, 17
	vslo	16, 16, 17		# mask

	lxvb16x 32+19, 0, 14		# load partial block
	sldi	15, 12, 3
	mtvsrdd	32+17, 0, 15
	vsro	19, 19, 17		# 0x00..xxxx??..??
	sldi	15, 17, 3
	mtvsrdd	32+17, 0, 15
	vsro	19, 19, 17		# 0x00..xxxx
	vslo	19, 19, 17		# shift back to form 0x00..xxxx00..00

	# AES crypt partial
	xxlxor	32+15, 32+30, 0
	lwz	23, 240(6)	# n rounds
	addi	22, 23, -1	# loop - 1
	mtctr	22
	addi	10, 6, 16

__Loop_aes_cpstate:
	lxv	32+1, 0(10)
	vcipher	15, 15, 1
	addi	10, 10, 16
	bdnz	__Loop_aes_cpstate
	lxv	32+1, 0(10)		# last round key
	vcipherlast 15, 15, 1

	vxor	15, 15, 19
	vand	15, 15, 16

	# AES crypt output v15
	# Write partial
	li	10, 224
	stxvb16x 15+32, 10, 1		# write v15 to stack
	addi	10, 1, 223
	add	10, 10, 12		# add offset
	addi	15, 9, -1
        mtctr	21			# partial block len
__Write_combine_partial:
        lbzu	22, 1(10)
	stbu	22, 1(15)
        bdnz	__Write_combine_partial

	add	14, 14, 21
	add	11, 11, 21
	add	9, 9, 21
	sub	5, 5, 21

	# Encrypt/Decrypt?
	cmpdi	24, 0			# decrypt?
	bne	__Encrypt_combine_partial
	vmr	15, 19		# decrypt using the input block

__Encrypt_combine_partial:
	#
	# Update partial flag and combine ghash.
__Update_partial_ghash:
	li	10, 64
	lxvb16x 32+17, 10, 7		# load previous pblock
	add	12, 12, 21		# combined pprocessed
	vxor	15, 15, 17		# combined pblock

	cmpdi	12, 16
	beq	__Clear_partial_flag
	std	12, 56(7)		# update partial len
	stxvb16x 32+15, 10, 7		# Update current pblock
	blr

__Clear_partial_flag:
	li	12, 0
	std	12, 56(7)
	# Update IV and ghash here
	vadduwm	30, 30, 31		# increase IV
	stxvb16x 32+30, 0, 7		# update IV

	# v15 either is either (input blockor encrypted)^(AES state)
	vxor	15, 15, 0
	PPC_GHASH1x 0, 15
	stxvb16x 32+0, 10, 7		# update pblock for debug?
	stxvb16x 32+0, 0, 8		# update Xi
	blr
SYM_FUNC_END(__Combine_partial)

################################################################################
# gcm_update(iv, Xi) - compute last hash
#
################################################################################
SYM_FUNC_START(gcm_update)

	ld	10, 56(3)
	cmpdi	10, 0
	beq	__no_update

	lxvb16x	32, 0, 4	# load Xi
	# load Hash - h^4, h^3, h^2, h
	li	10, 32
	lxvd2x	2+32, 10, 4	# H Poli
	li	10, 48
	lxvd2x	3+32, 10, 4	# Hl
	li	10, 64
	lxvd2x	4+32, 10, 4	# H
	li	10, 80
	lxvd2x	5+32, 10, 4	# Hh

	addis	11, 2, permx@toc@ha
	addi	11, 11, permx@toc@l
	lxv	10, 0(11)	# vs10: vpermxor vector

	li	9, 64
	lxvb16x 32+6, 9, 3		# load pblock
	vxor	6, 6, 0

	vxor	1, 1, 1
	vpmsumd	12, 3, 6		# L
	vpmsumd	13, 4, 6		# M
	vpmsumd	14, 5, 6		# H
	vpmsumd	17, 12, 2		# reduction
	vsldoi	15, 13, 1, 8		# mL
	vsldoi	16, 1, 13, 8		# mH
	vxor	12, 12, 15		# LL + LL
	vxor	14, 14, 16		# HH + HH
	xxlor	32+15, 10, 10
	vpermxor 12, 12, 17, 15
	vsldoi	13, 12, 12, 8		# swap
	vpmsumd	12, 12, 2		# reduction
	vxor	13, 13, 14
	vxor	7, 12, 13

	#vxor	0, 0, 0
	#stxvb16x 32+0, 9, 3
	li	10, 0
	std	10, 56(3)
	stxvb16x 32+7, 0, 4

__no_update:
	blr
SYM_FUNC_END(gcm_update)

################################################################################
# aes_p10_gcm_encrypt (const void *inp, void *out, size_t len,
#               const char *rk, unsigned char iv[16], void *Xip);
#
#    r3 - inp
#    r4 - out
#    r5 - len
#    r6 - AES round keys
#    r7 - iv and other data
#    r8 - Xi, HPoli, hash keys
#
#    rounds is at offset 240 in rk
#    Xi is at 0 in gcm_table (Xip).
#
################################################################################
SYM_FUNC_START(aes_p10_gcm_encrypt)

	cmpdi	5, 0
	ble	__Invalid_msg_len

	SAVE_REGS
	LOAD_HASH_TABLE

	# initialize ICB: GHASH( IV ), IV - r7
	lxvb16x	30+32, 0, 7	# load IV  - v30

	mr	14, 3
	mr	9, 4

	# counter 1
	vxor	31, 31, 31
	vspltisb 22, 1
	vsldoi	31, 31, 22,1	# counter 1

	addis	11, 2, permx@toc@ha
	addi	11, 11, permx@toc@l
	lxv	10, 0(11)	# vs10: vpermxor vector
	li	11, 0

	# load 9 round keys to VSR
	lxv	0, 0(6)			# round key 0
	lxv	1, 16(6)		# round key 1
	lxv	2, 32(6)		# round key 2
	lxv	3, 48(6)		# round key 3
	lxv	4, 64(6)		# round key 4
	lxv	5, 80(6)		# round key 5
	lxv	6, 96(6)		# round key 6
	lxv	7, 112(6)		# round key 7
	lxv	8, 128(6)		# round key 8

	# load rounds - 10 (128), 12 (192), 14 (256)
	lwz	23, 240(6)		# n rounds
	li	24, 1			# encrypt

__Process_encrypt:
	#
	# Process different blocks
	#
	ld	12, 56(7)
	cmpdi	12, 0
	bgt	__Do_combine_enc
	cmpdi	5, 128
	blt	__Process_more_enc

#
# Process 8x AES/GCM blocks
#
__Process_8x_enc:
	# 8x blcoks
	li	10, 128
	divdu	12, 5, 10	# n 128 bytes-blocks

	addi	12, 12, -1	# loop - 1

	vmr	15, 30		# first state: IV
	vadduwm	16, 15, 31	# state + counter
	vadduwm	17, 16, 31
	vadduwm	18, 17, 31
	vadduwm	19, 18, 31
	vadduwm	20, 19, 31
	vadduwm	21, 20, 31
	vadduwm	22, 21, 31
	xxlor	9, 32+22, 32+22	# save last state

	# vxor  state, state, w # addroundkey
	xxlor	32+29, 0, 0
        vxor    15, 15, 29      # IV + round key - add round key 0
	vxor	16, 16, 29
	vxor	17, 17, 29
	vxor	18, 18, 29
	vxor	19, 19, 29
	vxor	20, 20, 29
	vxor	21, 21, 29
	vxor	22, 22, 29

	li	15, 16
	li	16, 32
	li	17, 48
	li	18, 64
	li	19, 80
	li	20, 96
	li	21, 112

	#
	# Pre-compute first 8 AES state and leave 1/3/5 more rounds
	# for the loop.
	#
	addi	22, 23, -9		# process 8 keys
	mtctr	22			# AES key loop
	addi	10, 6, 144

	LOOP_8AES_STATE			# process 8 AES keys

__PreLoop_aes_state:
	lxv	32+1, 0(10)		# round key
	AES_CIPHER_8x vcipher 15 1
	addi	10, 10, 16
	bdnz	__PreLoop_aes_state
	lxv	32+1, 0(10)		# last round key (v1)

	cmpdi	12, 0			# Only one loop (8 block)
	beq	__Finish_ghash

#
# Loop 8x blocks and compute ghash
#
__Loop_8x_block_enc:
	vcipherlast     15, 15, 1
	vcipherlast     16, 16, 1
	vcipherlast     17, 17, 1
	vcipherlast     18, 18, 1
	vcipherlast     19, 19, 1
	vcipherlast     20, 20, 1
	vcipherlast     21, 21, 1
	vcipherlast     22, 22, 1

	lxvb16x	32+23, 0, 14	# load block
	lxvb16x	32+24, 15, 14	# load block
	lxvb16x	32+25, 16, 14	# load block
	lxvb16x	32+26, 17, 14	# load block
	lxvb16x	32+27, 18, 14	# load block
	lxvb16x	32+28, 19, 14	# load block
	lxvb16x	32+29, 20, 14	# load block
	lxvb16x	32+30, 21, 14	# load block
	addi	14, 14, 128

	vxor	15, 15, 23
	vxor	16, 16, 24
	vxor	17, 17, 25
	vxor	18, 18, 26
	vxor	19, 19, 27
	vxor	20, 20, 28
	vxor	21, 21, 29
	vxor	22, 22, 30

	stxvb16x 47, 0, 9	# store output
	stxvb16x 48, 15, 9	# store output
	stxvb16x 49, 16, 9	# store output
	stxvb16x 50, 17, 9	# store output
	stxvb16x 51, 18, 9	# store output
	stxvb16x 52, 19, 9	# store output
	stxvb16x 53, 20, 9	# store output
	stxvb16x 54, 21, 9	# store output
	addi	9, 9, 128

	# ghash here
	vxor	15, 15, 0
	PPC_GHASH4x 0, 15, 16, 17, 18

	vxor	19, 19, 0
	PPC_GHASH4x 0, 19, 20, 21, 22

	xxlor	32+15, 9, 9		# last state
	vadduwm 15, 15, 31		# state + counter
	vadduwm 16, 15, 31
	vadduwm 17, 16, 31
	vadduwm 18, 17, 31
	vadduwm 19, 18, 31
	vadduwm 20, 19, 31
	vadduwm 21, 20, 31
	vadduwm 22, 21, 31
	xxlor	9, 32+22, 32+22		# save last state

	xxlor	32+27, 0, 0		# restore roundkey 0
        vxor    15, 15, 27		# IV + round key - add round key 0
	vxor	16, 16, 27
	vxor	17, 17, 27
	vxor	18, 18, 27
	vxor	19, 19, 27
	vxor	20, 20, 27
	vxor	21, 21, 27
	vxor	22, 22, 27

	addi    5, 5, -128
	addi    11, 11, 128

	LOOP_8AES_STATE			# process 8 AES keys
	mtctr	22			# AES key loop
	addi	10, 6, 144
__LastLoop_aes_state:
	lxv	32+1, 0(10)		# round key
	AES_CIPHER_8x vcipher 15 1
	addi	10, 10, 16
	bdnz	__LastLoop_aes_state
	lxv	32+1, 0(10)		# last round key (v1)

	addi	12, 12, -1
	cmpdi	12, 0
	bne	__Loop_8x_block_enc

__Finish_ghash:
	vcipherlast     15, 15, 1
	vcipherlast     16, 16, 1
	vcipherlast     17, 17, 1
	vcipherlast     18, 18, 1
	vcipherlast     19, 19, 1
	vcipherlast     20, 20, 1
	vcipherlast     21, 21, 1
	vcipherlast     22, 22, 1

	lxvb16x	32+23, 0, 14	# load block
	lxvb16x	32+24, 15, 14	# load block
	lxvb16x	32+25, 16, 14	# load block
	lxvb16x	32+26, 17, 14	# load block
	lxvb16x	32+27, 18, 14	# load block
	lxvb16x	32+28, 19, 14	# load block
	lxvb16x	32+29, 20, 14	# load block
	lxvb16x	32+30, 21, 14	# load block
	addi	14, 14, 128

	vxor	15, 15, 23
	vxor	16, 16, 24
	vxor	17, 17, 25
	vxor	18, 18, 26
	vxor	19, 19, 27
	vxor	20, 20, 28
	vxor	21, 21, 29
	vxor	22, 22, 30

	stxvb16x 47, 0, 9	# store output
	stxvb16x 48, 15, 9	# store output
	stxvb16x 49, 16, 9	# store output
	stxvb16x 50, 17, 9	# store output
	stxvb16x 51, 18, 9	# store output
	stxvb16x 52, 19, 9	# store output
	stxvb16x 53, 20, 9	# store output
	stxvb16x 54, 21, 9	# store output
	addi	9, 9, 128

	vxor	15, 15, 0
	PPC_GHASH4x 0, 15, 16, 17, 18

	vxor	19, 19, 0
	PPC_GHASH4x 0, 19, 20, 21, 22

	xxlor	30+32, 9, 9		# last ctr
	vadduwm	30, 30, 31		# increase ctr
	stxvb16x 32+30, 0, 7		# update IV
	stxvb16x 32+0, 0, 8		# update Xi

	addi    5, 5, -128
	addi    11, 11, 128

	#
	# Done 8x blocks
	#

	cmpdi   5, 0
	beq     aes_gcm_out

__Process_more_enc:
	li	24, 1			# encrypt
	bl	aes_gcm_crypt_1x
	cmpdi   5, 0
	beq     aes_gcm_out

	bl	__Process_partial
	cmpdi   5, 0
	beq     aes_gcm_out
__Do_combine_enc:
	bl	__Combine_partial
	cmpdi	5, 0
	bgt	__Process_encrypt
	b	aes_gcm_out

SYM_FUNC_END(aes_p10_gcm_encrypt)

################################################################################
# aes_p10_gcm_decrypt (const void *inp, void *out, size_t len,
#               const char *rk, unsigned char iv[16], void *Xip);
# 8x Decrypt
#
################################################################################
SYM_FUNC_START(aes_p10_gcm_decrypt)

	cmpdi	5, 0
	ble	__Invalid_msg_len

	SAVE_REGS
	LOAD_HASH_TABLE

	# initialize ICB: GHASH( IV ), IV - r7
	lxvb16x	30+32, 0, 7	# load IV  - v30

	mr	14, 3
	mr	9, 4

	# counter 1
	vxor	31, 31, 31
	vspltisb 22, 1
	vsldoi	31, 31, 22,1	# counter 1

	addis	11, 2, permx@toc@ha
	addi	11, 11, permx@toc@l
	lxv	10, 0(11)	# vs10: vpermxor vector
	li	11, 0

	# load 9 round keys to VSR
	lxv	0, 0(6)			# round key 0
	lxv	1, 16(6)		# round key 1
	lxv	2, 32(6)		# round key 2
	lxv	3, 48(6)		# round key 3
	lxv	4, 64(6)		# round key 4
	lxv	5, 80(6)		# round key 5
	lxv	6, 96(6)		# round key 6
	lxv	7, 112(6)		# round key 7
	lxv	8, 128(6)		# round key 8

	# load rounds - 10 (128), 12 (192), 14 (256)
	lwz	23, 240(6)		# n rounds
	li	24, 0			# decrypt

__Process_decrypt:
	#
	# Process different blocks
	#
	ld	12, 56(7)
	cmpdi	12, 0
	bgt	__Do_combine_dec
	cmpdi	5, 128
	blt	__Process_more_dec

#
# Process 8x AES/GCM blocks
#
__Process_8x_dec:
	# 8x blcoks
	li	10, 128
	divdu	12, 5, 10	# n 128 bytes-blocks

	addi	12, 12, -1	# loop - 1

	vmr	15, 30		# first state: IV
	vadduwm	16, 15, 31	# state + counter
	vadduwm	17, 16, 31
	vadduwm	18, 17, 31
	vadduwm	19, 18, 31
	vadduwm	20, 19, 31
	vadduwm	21, 20, 31
	vadduwm	22, 21, 31
	xxlor	9, 32+22, 32+22	# save last state

	# vxor  state, state, w # addroundkey
	xxlor	32+29, 0, 0
        vxor    15, 15, 29      # IV + round key - add round key 0
	vxor	16, 16, 29
	vxor	17, 17, 29
	vxor	18, 18, 29
	vxor	19, 19, 29
	vxor	20, 20, 29
	vxor	21, 21, 29
	vxor	22, 22, 29

	li	15, 16
	li	16, 32
	li	17, 48
	li	18, 64
	li	19, 80
	li	20, 96
	li	21, 112

	#
	# Pre-compute first 8 AES state and leave 1/3/5 more rounds
	# for the loop.
	#
	addi	22, 23, -9		# process 8 keys
	mtctr	22			# AES key loop
	addi	10, 6, 144

	LOOP_8AES_STATE			# process 8 AES keys

__PreLoop_aes_state_dec:
	lxv	32+1, 0(10)		# round key
	AES_CIPHER_8x vcipher 15 1
	addi	10, 10, 16
	bdnz	__PreLoop_aes_state_dec
	lxv	32+1, 0(10)		# last round key (v1)

	cmpdi	12, 0			# Only one loop (8 block)
	beq	__Finish_ghash_dec

#
# Loop 8x blocks and compute ghash
#
__Loop_8x_block_dec:
	vcipherlast     15, 15, 1
	vcipherlast     16, 16, 1
	vcipherlast     17, 17, 1
	vcipherlast     18, 18, 1
	vcipherlast     19, 19, 1
	vcipherlast     20, 20, 1
	vcipherlast     21, 21, 1
	vcipherlast     22, 22, 1

	lxvb16x	32+23, 0, 14	# load block
	lxvb16x	32+24, 15, 14	# load block
	lxvb16x	32+25, 16, 14	# load block
	lxvb16x	32+26, 17, 14	# load block
	lxvb16x	32+27, 18, 14	# load block
	lxvb16x	32+28, 19, 14	# load block
	lxvb16x	32+29, 20, 14	# load block
	lxvb16x	32+30, 21, 14	# load block
	addi	14, 14, 128

	vxor	15, 15, 23
	vxor	16, 16, 24
	vxor	17, 17, 25
	vxor	18, 18, 26
	vxor	19, 19, 27
	vxor	20, 20, 28
	vxor	21, 21, 29
	vxor	22, 22, 30

	stxvb16x 47, 0, 9	# store output
	stxvb16x 48, 15, 9	# store output
	stxvb16x 49, 16, 9	# store output
	stxvb16x 50, 17, 9	# store output
	stxvb16x 51, 18, 9	# store output
	stxvb16x 52, 19, 9	# store output
	stxvb16x 53, 20, 9	# store output
	stxvb16x 54, 21, 9	# store output

	addi	9, 9, 128

	vmr	15, 23
	vmr	16, 24
	vmr	17, 25
	vmr	18, 26
	vmr	19, 27
	vmr	20, 28
	vmr	21, 29
	vmr	22, 30

	# ghash here
	vxor	15, 15, 0
	PPC_GHASH4x 0, 15, 16, 17, 18

	vxor	19, 19, 0
	PPC_GHASH4x 0, 19, 20, 21, 22

	xxlor	32+15, 9, 9		# last state
	vadduwm 15, 15, 31		# state + counter
	vadduwm 16, 15, 31
	vadduwm 17, 16, 31
	vadduwm 18, 17, 31
	vadduwm 19, 18, 31
	vadduwm 20, 19, 31
	vadduwm 21, 20, 31
	vadduwm 22, 21, 31
	xxlor	9, 32+22, 32+22		# save last state

	xxlor	32+27, 0, 0		# restore roundkey 0
        vxor    15, 15, 27		# IV + round key - add round key 0
	vxor	16, 16, 27
	vxor	17, 17, 27
	vxor	18, 18, 27
	vxor	19, 19, 27
	vxor	20, 20, 27
	vxor	21, 21, 27
	vxor	22, 22, 27

	addi    5, 5, -128
	addi    11, 11, 128

	LOOP_8AES_STATE			# process 8 AES keys
	mtctr	22			# AES key loop
	addi	10, 6, 144
__LastLoop_aes_state_dec:
	lxv	32+1, 0(10)		# round key
	AES_CIPHER_8x vcipher 15 1
	addi	10, 10, 16
	bdnz	__LastLoop_aes_state_dec
	lxv	32+1, 0(10)		# last round key (v1)

	addi	12, 12, -1
	cmpdi	12, 0
	bne	__Loop_8x_block_dec

__Finish_ghash_dec:
	vcipherlast     15, 15, 1
	vcipherlast     16, 16, 1
	vcipherlast     17, 17, 1
	vcipherlast     18, 18, 1
	vcipherlast     19, 19, 1
	vcipherlast     20, 20, 1
	vcipherlast     21, 21, 1
	vcipherlast     22, 22, 1

	lxvb16x	32+23, 0, 14	# load block
	lxvb16x	32+24, 15, 14	# load block
	lxvb16x	32+25, 16, 14	# load block
	lxvb16x	32+26, 17, 14	# load block
	lxvb16x	32+27, 18, 14	# load block
	lxvb16x	32+28, 19, 14	# load block
	lxvb16x	32+29, 20, 14	# load block
	lxvb16x	32+30, 21, 14	# load block
	addi	14, 14, 128

	vxor	15, 15, 23
	vxor	16, 16, 24
	vxor	17, 17, 25
	vxor	18, 18, 26
	vxor	19, 19, 27
	vxor	20, 20, 28
	vxor	21, 21, 29
	vxor	22, 22, 30

	stxvb16x 47, 0, 9	# store output
	stxvb16x 48, 15, 9	# store output
	stxvb16x 49, 16, 9	# store output
	stxvb16x 50, 17, 9	# store output
	stxvb16x 51, 18, 9	# store output
	stxvb16x 52, 19, 9	# store output
	stxvb16x 53, 20, 9	# store output
	stxvb16x 54, 21, 9	# store output
	addi	9, 9, 128

	#vmr	15, 23
	vxor	15, 23, 0
	vmr	16, 24
	vmr	17, 25
	vmr	18, 26
	vmr	19, 27
	vmr	20, 28
	vmr	21, 29
	vmr	22, 30

	#vxor	15, 15, 0
	PPC_GHASH4x 0, 15, 16, 17, 18

	vxor	19, 19, 0
	PPC_GHASH4x 0, 19, 20, 21, 22

	xxlor	30+32, 9, 9		# last ctr
	vadduwm	30, 30, 31		# increase ctr
	stxvb16x 32+30, 0, 7		# update IV
	stxvb16x 32+0, 0, 8		# update Xi

	addi    5, 5, -128
	addi    11, 11, 128

	#
	# Done 8x blocks
	#

	cmpdi   5, 0
	beq     aes_gcm_out

__Process_more_dec:
	li	24, 0			# decrypt
	bl	aes_gcm_crypt_1x
	cmpdi   5, 0
	beq     aes_gcm_out

	bl	__Process_partial
	cmpdi   5, 0
	beq     aes_gcm_out
__Do_combine_dec:
	bl	__Combine_partial
	cmpdi	5, 0
	bgt	__Process_decrypt
	b	aes_gcm_out
SYM_FUNC_END(aes_p10_gcm_decrypt)

SYM_FUNC_START_LOCAL(aes_gcm_out)

	mr	3, 11			# return count

	RESTORE_REGS
	blr

__Invalid_msg_len:
	li	3, 0
	blr
SYM_FUNC_END(aes_gcm_out)

SYM_DATA_START_LOCAL(PERMX)
.align 4
# for vector permute and xor
permx:
.long 0x4c5d6e7f, 0x08192a3b, 0xc4d5e6f7, 0x8091a2b3
SYM_DATA_END(permx)