sha1-ssse3-and-avx.S

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * This is a SIMD SHA-1 implementation. It requires the Intel(R) Supplemental
 * SSE3 instruction set extensions introduced in Intel Core Microarchitecture
 * processors. CPUs supporting Intel(R) AVX extensions will get an additional
 * boost.
 *
 * This work was inspired by the vectorized implementation of Dean Gaudet.
 * Additional information on it can be found at:
 *    http://www.arctic.org/~dean/crypto/sha1.html
 *
 * It was improved upon with more efficient vectorization of the message
 * scheduling. This implementation has also been optimized for all current and
 * several future generations of Intel CPUs.
 *
 * See this article for more information about the implementation details:
 *   http://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1/
 *
 * Copyright (C) 2010, Intel Corp.
 *   Authors: Maxim Locktyukhin <[email protected]>
 *            Ronen Zohar <[email protected]>
 *
 * Converted to AT&T syntax and adapted for inclusion in the Linux kernel:
 *   Author: Mathias Krause <[email protected]>
 */

#include <linux/linkage.h>

#define CTX	%rdi	// arg1
#define BUF	%rsi	// arg2
#define CNT	%rdx	// arg3

#define REG_A	%ecx
#define REG_B	%esi
#define REG_C	%edi
#define REG_D	%r12d
#define REG_E	%edx

#define REG_T1	%eax
#define REG_T2	%ebx

#define K_BASE		%r8
#define HASH_PTR	%r9
#define BUFFER_PTR	%r10
#define BUFFER_END	%r11

#define W_TMP1	%xmm0
#define W_TMP2	%xmm9

#define W0	%xmm1
#define W4	%xmm2
#define W8	%xmm3
#define W12	%xmm4
#define W16	%xmm5
#define W20	%xmm6
#define W24	%xmm7
#define W28	%xmm8

#define XMM_SHUFB_BSWAP	%xmm10

/* we keep window of 64 w[i]+K pre-calculated values in a circular buffer */
#define WK(t)	(((t) & 15) * 4)(%rsp)
#define W_PRECALC_AHEAD	16

/*
 * This macro implements the SHA-1 function's body for single 64-byte block
 * param: function's name
 */
.macro SHA1_VECTOR_ASM  name
	SYM_FUNC_START(\name)

	push	%rbx
	push	%r12
	push	%rbp
	mov	%rsp, %rbp

	sub	$64, %rsp		# allocate workspace
	and	$~15, %rsp		# align stack

	mov	CTX, HASH_PTR
	mov	BUF, BUFFER_PTR

	shl	$6, CNT			# multiply by 64
	add	BUF, CNT
	mov	CNT, BUFFER_END

	lea	K_XMM_AR(%rip), K_BASE
	xmm_mov	BSWAP_SHUFB_CTL(%rip), XMM_SHUFB_BSWAP

	SHA1_PIPELINED_MAIN_BODY

	# cleanup workspace
	mov	$8, %ecx
	mov	%rsp, %rdi
	xor	%eax, %eax
	rep stosq

	mov	%rbp, %rsp		# deallocate workspace
	pop	%rbp
	pop	%r12
	pop	%rbx
	RET

	SYM_FUNC_END(\name)
.endm

/*
 * This macro implements 80 rounds of SHA-1 for one 64-byte block
 */
.macro SHA1_PIPELINED_MAIN_BODY
	INIT_REGALLOC

	mov	  (HASH_PTR), A
	mov	 4(HASH_PTR), B
	mov	 8(HASH_PTR), C
	mov	12(HASH_PTR), D
	mov	16(HASH_PTR), E

  .set i, 0
  .rept W_PRECALC_AHEAD
	W_PRECALC i
    .set i, (i+1)
  .endr

.align 4
1:
	RR F1,A,B,C,D,E,0
	RR F1,D,E,A,B,C,2
	RR F1,B,C,D,E,A,4
	RR F1,E,A,B,C,D,6
	RR F1,C,D,E,A,B,8

	RR F1,A,B,C,D,E,10
	RR F1,D,E,A,B,C,12
	RR F1,B,C,D,E,A,14
	RR F1,E,A,B,C,D,16
	RR F1,C,D,E,A,B,18

	RR F2,A,B,C,D,E,20
	RR F2,D,E,A,B,C,22
	RR F2,B,C,D,E,A,24
	RR F2,E,A,B,C,D,26
	RR F2,C,D,E,A,B,28

	RR F2,A,B,C,D,E,30
	RR F2,D,E,A,B,C,32
	RR F2,B,C,D,E,A,34
	RR F2,E,A,B,C,D,36
	RR F2,C,D,E,A,B,38

	RR F3,A,B,C,D,E,40
	RR F3,D,E,A,B,C,42
	RR F3,B,C,D,E,A,44
	RR F3,E,A,B,C,D,46
	RR F3,C,D,E,A,B,48

	RR F3,A,B,C,D,E,50
	RR F3,D,E,A,B,C,52
	RR F3,B,C,D,E,A,54
	RR F3,E,A,B,C,D,56
	RR F3,C,D,E,A,B,58

	add	$64, BUFFER_PTR		# move to the next 64-byte block
	cmp	BUFFER_END, BUFFER_PTR	# if the current is the last one use
	cmovae	K_BASE, BUFFER_PTR	# dummy source to avoid buffer overrun

	RR F4,A,B,C,D,E,60
	RR F4,D,E,A,B,C,62
	RR F4,B,C,D,E,A,64
	RR F4,E,A,B,C,D,66
	RR F4,C,D,E,A,B,68

	RR F4,A,B,C,D,E,70
	RR F4,D,E,A,B,C,72
	RR F4,B,C,D,E,A,74
	RR F4,E,A,B,C,D,76
	RR F4,C,D,E,A,B,78

	UPDATE_HASH   (HASH_PTR), A
	UPDATE_HASH  4(HASH_PTR), B
	UPDATE_HASH  8(HASH_PTR), C
	UPDATE_HASH 12(HASH_PTR), D
	UPDATE_HASH 16(HASH_PTR), E

	RESTORE_RENAMED_REGS
	cmp	K_BASE, BUFFER_PTR	# K_BASE means, we reached the end
	jne	1b
.endm

.macro INIT_REGALLOC
  .set A, REG_A
  .set B, REG_B
  .set C, REG_C
  .set D, REG_D
  .set E, REG_E
  .set T1, REG_T1
  .set T2, REG_T2
.endm

.macro RESTORE_RENAMED_REGS
	# order is important (REG_C is where it should be)
	mov	B, REG_B
	mov	D, REG_D
	mov	A, REG_A
	mov	E, REG_E
.endm

.macro SWAP_REG_NAMES  a, b
  .set _T, \a
  .set \a, \b
  .set \b, _T
.endm

.macro F1  b, c, d
	mov	\c, T1
	SWAP_REG_NAMES \c, T1
	xor	\d, T1
	and	\b, T1
	xor	\d, T1
.endm

.macro F2  b, c, d
	mov	\d, T1
	SWAP_REG_NAMES \d, T1
	xor	\c, T1
	xor	\b, T1
.endm

.macro F3  b, c ,d
	mov	\c, T1
	SWAP_REG_NAMES \c, T1
	mov	\b, T2
	or	\b, T1
	and	\c, T2
	and	\d, T1
	or	T2, T1
.endm

.macro F4  b, c, d
	F2 \b, \c, \d
.endm

.macro UPDATE_HASH  hash, val
	add	\hash, \val
	mov	\val, \hash
.endm

/*
 * RR does two rounds of SHA-1 back to back with W[] pre-calc
 *   t1 = F(b, c, d);   e += w(i)
 *   e += t1;           b <<= 30;   d  += w(i+1);
 *   t1 = F(a, b, c);
 *   d += t1;           a <<= 5;
 *   e += a;
 *   t1 = e;            a >>= 7;
 *   t1 <<= 5;
 *   d += t1;
 */
.macro RR  F, a, b, c, d, e, round
	add	WK(\round), \e
	\F   \b, \c, \d		# t1 = F(b, c, d);
	W_PRECALC (\round + W_PRECALC_AHEAD)
	rol	$30, \b
	add	T1, \e
	add	WK(\round + 1), \d

	\F   \a, \b, \c
	W_PRECALC (\round + W_PRECALC_AHEAD + 1)
	rol	$5, \a
	add	\a, \e
	add	T1, \d
	ror	$7, \a		# (a <<r 5) >>r 7) => a <<r 30)

	mov	\e, T1
	SWAP_REG_NAMES \e, T1

	rol	$5, T1
	add	T1, \d

	# write:  \a, \b
	# rotate: \a<=\d, \b<=\e, \c<=\a, \d<=\b, \e<=\c
.endm

.macro W_PRECALC  r
  .set i, \r

  .if (i < 20)
    .set K_XMM, 0
  .elseif (i < 40)
    .set K_XMM, 16
  .elseif (i < 60)
    .set K_XMM, 32
  .elseif (i < 80)
    .set K_XMM, 48
  .endif

  .if ((i < 16) || ((i >= 80) && (i < (80 + W_PRECALC_AHEAD))))
    .set i, ((\r) % 80)	    # pre-compute for the next iteration
    .if (i == 0)
	W_PRECALC_RESET
    .endif
	W_PRECALC_00_15
  .elseif (i<32)
	W_PRECALC_16_31
  .elseif (i < 80)   // rounds 32-79
	W_PRECALC_32_79
  .endif
.endm

.macro W_PRECALC_RESET
  .set W,          W0
  .set W_minus_04, W4
  .set W_minus_08, W8
  .set W_minus_12, W12
  .set W_minus_16, W16
  .set W_minus_20, W20
  .set W_minus_24, W24
  .set W_minus_28, W28
  .set W_minus_32, W
.endm

.macro W_PRECALC_ROTATE
  .set W_minus_32, W_minus_28
  .set W_minus_28, W_minus_24
  .set W_minus_24, W_minus_20
  .set W_minus_20, W_minus_16
  .set W_minus_16, W_minus_12
  .set W_minus_12, W_minus_08
  .set W_minus_08, W_minus_04
  .set W_minus_04, W
  .set W,          W_minus_32
.endm

.macro W_PRECALC_SSSE3

.macro W_PRECALC_00_15
	W_PRECALC_00_15_SSSE3
.endm
.macro W_PRECALC_16_31
	W_PRECALC_16_31_SSSE3
.endm
.macro W_PRECALC_32_79
	W_PRECALC_32_79_SSSE3
.endm

/* message scheduling pre-compute for rounds 0-15 */
.macro W_PRECALC_00_15_SSSE3
  .if ((i & 3) == 0)
	movdqu	(i*4)(BUFFER_PTR), W_TMP1
  .elseif ((i & 3) == 1)
	pshufb	XMM_SHUFB_BSWAP, W_TMP1
	movdqa	W_TMP1, W
  .elseif ((i & 3) == 2)
	paddd	(K_BASE), W_TMP1
  .elseif ((i & 3) == 3)
	movdqa  W_TMP1, WK(i&~3)
	W_PRECALC_ROTATE
  .endif
.endm

/* message scheduling pre-compute for rounds 16-31
 *
 * - calculating last 32 w[i] values in 8 XMM registers
 * - pre-calculate K+w[i] values and store to mem, for later load by ALU add
 *   instruction
 *
 * some "heavy-lifting" vectorization for rounds 16-31 due to w[i]->w[i-3]
 * dependency, but improves for 32-79
 */
.macro W_PRECALC_16_31_SSSE3
  # blended scheduling of vector and scalar instruction streams, one 4-wide
  # vector iteration / 4 scalar rounds
  .if ((i & 3) == 0)
	movdqa	W_minus_12, W
	palignr	$8, W_minus_16, W	# w[i-14]
	movdqa	W_minus_04, W_TMP1
	psrldq	$4, W_TMP1		# w[i-3]
	pxor	W_minus_08, W
  .elseif ((i & 3) == 1)
	pxor	W_minus_16, W_TMP1
	pxor	W_TMP1, W
	movdqa	W, W_TMP2
	movdqa	W, W_TMP1
	pslldq	$12, W_TMP2
  .elseif ((i & 3) == 2)
	psrld	$31, W
	pslld	$1, W_TMP1
	por	W, W_TMP1
	movdqa	W_TMP2, W
	psrld	$30, W_TMP2
	pslld	$2, W
  .elseif ((i & 3) == 3)
	pxor	W, W_TMP1
	pxor	W_TMP2, W_TMP1
	movdqa	W_TMP1, W
	paddd	K_XMM(K_BASE), W_TMP1
	movdqa	W_TMP1, WK(i&~3)
	W_PRECALC_ROTATE
  .endif
.endm

/* message scheduling pre-compute for rounds 32-79
 *
 * in SHA-1 specification: w[i] = (w[i-3] ^ w[i-8]  ^ w[i-14] ^ w[i-16]) rol 1
 * instead we do equal:    w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
 * allows more efficient vectorization since w[i]=>w[i-3] dependency is broken
 */
.macro W_PRECALC_32_79_SSSE3
  .if ((i & 3) == 0)
	movdqa	W_minus_04, W_TMP1
	pxor	W_minus_28, W		# W is W_minus_32 before xor
	palignr	$8, W_minus_08, W_TMP1
  .elseif ((i & 3) == 1)
	pxor	W_minus_16, W
	pxor	W_TMP1, W
	movdqa	W, W_TMP1
  .elseif ((i & 3) == 2)
	psrld	$30, W
	pslld	$2, W_TMP1
	por	W, W_TMP1
  .elseif ((i & 3) == 3)
	movdqa	W_TMP1, W
	paddd	K_XMM(K_BASE), W_TMP1
	movdqa	W_TMP1, WK(i&~3)
	W_PRECALC_ROTATE
  .endif
.endm

.endm		// W_PRECALC_SSSE3


#define K1	0x5a827999
#define K2	0x6ed9eba1
#define K3	0x8f1bbcdc
#define K4	0xca62c1d6

.section .rodata
.align 16

K_XMM_AR:
	.long K1, K1, K1, K1
	.long K2, K2, K2, K2
	.long K3, K3, K3, K3
	.long K4, K4, K4, K4

BSWAP_SHUFB_CTL:
	.long 0x00010203
	.long 0x04050607
	.long 0x08090a0b
	.long 0x0c0d0e0f


.section .text

W_PRECALC_SSSE3
.macro xmm_mov a, b
	movdqu	\a,\b
.endm

/*
 * SSSE3 optimized implementation:
 *
 * void sha1_transform_ssse3(struct sha1_block_state *state,
 *			     const u8 *data, size_t nblocks);
 */
SHA1_VECTOR_ASM     sha1_transform_ssse3

.macro W_PRECALC_AVX

.purgem W_PRECALC_00_15
.macro  W_PRECALC_00_15
    W_PRECALC_00_15_AVX
.endm
.purgem W_PRECALC_16_31
.macro  W_PRECALC_16_31
    W_PRECALC_16_31_AVX
.endm
.purgem W_PRECALC_32_79
.macro  W_PRECALC_32_79
    W_PRECALC_32_79_AVX
.endm

.macro W_PRECALC_00_15_AVX
  .if ((i & 3) == 0)
	vmovdqu	(i*4)(BUFFER_PTR), W_TMP1
  .elseif ((i & 3) == 1)
	vpshufb	XMM_SHUFB_BSWAP, W_TMP1, W
  .elseif ((i & 3) == 2)
	vpaddd	(K_BASE), W, W_TMP1
  .elseif ((i & 3) == 3)
	vmovdqa	W_TMP1, WK(i&~3)
	W_PRECALC_ROTATE
  .endif
.endm

.macro W_PRECALC_16_31_AVX
  .if ((i & 3) == 0)
	vpalignr $8, W_minus_16, W_minus_12, W	# w[i-14]
	vpsrldq	$4, W_minus_04, W_TMP1		# w[i-3]
	vpxor	W_minus_08, W, W
	vpxor	W_minus_16, W_TMP1, W_TMP1
  .elseif ((i & 3) == 1)
	vpxor	W_TMP1, W, W
	vpslldq	$12, W, W_TMP2
	vpslld	$1, W, W_TMP1
  .elseif ((i & 3) == 2)
	vpsrld	$31, W, W
	vpor	W, W_TMP1, W_TMP1
	vpslld	$2, W_TMP2, W
	vpsrld	$30, W_TMP2, W_TMP2
  .elseif ((i & 3) == 3)
	vpxor	W, W_TMP1, W_TMP1
	vpxor	W_TMP2, W_TMP1, W
	vpaddd	K_XMM(K_BASE), W, W_TMP1
	vmovdqu	W_TMP1, WK(i&~3)
	W_PRECALC_ROTATE
  .endif
.endm

.macro W_PRECALC_32_79_AVX
  .if ((i & 3) == 0)
	vpalignr $8, W_minus_08, W_minus_04, W_TMP1
	vpxor	W_minus_28, W, W		# W is W_minus_32 before xor
  .elseif ((i & 3) == 1)
	vpxor	W_minus_16, W_TMP1, W_TMP1
	vpxor	W_TMP1, W, W
  .elseif ((i & 3) == 2)
	vpslld	$2, W, W_TMP1
	vpsrld	$30, W, W
	vpor	W, W_TMP1, W
  .elseif ((i & 3) == 3)
	vpaddd	K_XMM(K_BASE), W, W_TMP1
	vmovdqu	W_TMP1, WK(i&~3)
	W_PRECALC_ROTATE
  .endif
.endm

.endm    // W_PRECALC_AVX

W_PRECALC_AVX
.purgem xmm_mov
.macro xmm_mov a, b
	vmovdqu	\a,\b
.endm


/* AVX optimized implementation:
 * void sha1_transform_avx(struct sha1_block_state *state,
 *			   const u8 *data, size_t nblocks);
 */
SHA1_VECTOR_ASM     sha1_transform_avx