crc32-core.S

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Accelerated CRC32(C) using AArch64 CRC and PMULL instructions
 *
 * Copyright (C) 2016 - 2018 Linaro Ltd.
 * Copyright (C) 2024 Google LLC
 *
 * Author: Ard Biesheuvel <[email protected]>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

	.cpu		generic+crc+crypto

	.macro		bitle, reg
	.endm

	.macro		bitbe, reg
	rbit		\reg, \reg
	.endm

	.macro		bytele, reg
	.endm

	.macro		bytebe, reg
	rbit		\reg, \reg
	lsr		\reg, \reg, #24
	.endm

	.macro		hwordle, reg
CPU_BE(	rev16		\reg, \reg	)
	.endm

	.macro		hwordbe, reg
CPU_LE(	rev		\reg, \reg	)
	rbit		\reg, \reg
CPU_BE(	lsr		\reg, \reg, #16	)
	.endm

	.macro		le, regs:vararg
	.irp		r, \regs
CPU_BE(	rev		\r, \r		)
	.endr
	.endm

	.macro		be, regs:vararg
	.irp		r, \regs
CPU_LE(	rev		\r, \r		)
	.endr
	.irp		r, \regs
	rbit		\r, \r
	.endr
	.endm

	.macro		__crc32, c, order=le
	bit\order	w0
	cmp		x2, #16
	b.lt		8f			// less than 16 bytes

	and		x7, x2, #0x1f
	and		x2, x2, #~0x1f
	cbz		x7, 32f			// multiple of 32 bytes

	and		x8, x7, #0xf
	ldp		x3, x4, [x1]
	add		x8, x8, x1
	add		x1, x1, x7
	ldp		x5, x6, [x8]
	\order		x3, x4, x5, x6

	tst		x7, #8
	crc32\c\()x	w8, w0, x3
	csel		x3, x3, x4, eq
	csel		w0, w0, w8, eq
	tst		x7, #4
	lsr		x4, x3, #32
	crc32\c\()w	w8, w0, w3
	csel		x3, x3, x4, eq
	csel		w0, w0, w8, eq
	tst		x7, #2
	lsr		w4, w3, #16
	crc32\c\()h	w8, w0, w3
	csel		w3, w3, w4, eq
	csel		w0, w0, w8, eq
	tst		x7, #1
	crc32\c\()b	w8, w0, w3
	csel		w0, w0, w8, eq
	tst		x7, #16
	crc32\c\()x	w8, w0, x5
	crc32\c\()x	w8, w8, x6
	csel		w0, w0, w8, eq
	cbz		x2, 0f

32:	ldp		x3, x4, [x1], #32
	sub		x2, x2, #32
	ldp		x5, x6, [x1, #-16]
	\order		x3, x4, x5, x6
	crc32\c\()x	w0, w0, x3
	crc32\c\()x	w0, w0, x4
	crc32\c\()x	w0, w0, x5
	crc32\c\()x	w0, w0, x6
	cbnz		x2, 32b
0:	bit\order	w0
	ret

8:	tbz		x2, #3, 4f
	ldr		x3, [x1], #8
	\order		x3
	crc32\c\()x	w0, w0, x3
4:	tbz		x2, #2, 2f
	ldr		w3, [x1], #4
	\order		w3
	crc32\c\()w	w0, w0, w3
2:	tbz		x2, #1, 1f
	ldrh		w3, [x1], #2
	hword\order	w3
	crc32\c\()h	w0, w0, w3
1:	tbz		x2, #0, 0f
	ldrb		w3, [x1]
	byte\order	w3
	crc32\c\()b	w0, w0, w3
0:	bit\order	w0
	ret
	.endm

	.align		5
SYM_FUNC_START(crc32_le_arm64)
	__crc32
SYM_FUNC_END(crc32_le_arm64)

	.align		5
SYM_FUNC_START(crc32c_le_arm64)
	__crc32		c
SYM_FUNC_END(crc32c_le_arm64)

	.align		5
SYM_FUNC_START(crc32_be_arm64)
	__crc32		order=be
SYM_FUNC_END(crc32_be_arm64)

	in		.req	x1
	len		.req	x2

	/*
	 * w0: input CRC at entry, output CRC at exit
	 * x1: pointer to input buffer
	 * x2: length of input in bytes
	 */
	.macro		crc4way, insn, table, order=le
	bit\order	w0
	lsr		len, len, #6		// len := # of 64-byte blocks

	/* Process up to 64 blocks of 64 bytes at a time */
.La\@:	mov		x3, #64
	cmp		len, #64
	csel		x3, x3, len, hi		// x3 := min(len, 64)
	sub		len, len, x3

	/* Divide the input into 4 contiguous blocks */
	add		x4, x3, x3, lsl #1	// x4 :=  3 * x3
	add		x7, in, x3, lsl #4	// x7 := in + 16 * x3
	add		x8, in, x3, lsl #5	// x8 := in + 32 * x3
	add		x9, in, x4, lsl #4	// x9 := in + 16 * x4

	/* Load the folding coefficients from the lookup table */
	adr_l		x5, \table - 12		// entry 0 omitted
	add		x5, x5, x4, lsl #2	// x5 += 12 * x3
	ldp		s0, s1, [x5]
	ldr		s2, [x5, #8]

	/* Zero init partial CRCs for this iteration */
	mov		w4, wzr
	mov		w5, wzr
	mov		w6, wzr
	mov		x17, xzr

.Lb\@:	sub		x3, x3, #1
	\insn		w6, w6, x17
	ldp		x10, x11, [in], #16
	ldp		x12, x13, [x7], #16
	ldp		x14, x15, [x8], #16
	ldp		x16, x17, [x9], #16

	\order		x10, x11, x12, x13, x14, x15, x16, x17

	/* Apply the CRC transform to 4 16-byte blocks in parallel */
	\insn		w0, w0, x10
	\insn		w4, w4, x12
	\insn		w5, w5, x14
	\insn		w6, w6, x16
	\insn		w0, w0, x11
	\insn		w4, w4, x13
	\insn		w5, w5, x15
	cbnz		x3, .Lb\@

	/* Combine the 4 partial results into w0 */
	mov		v3.d[0], x0
	mov		v4.d[0], x4
	mov		v5.d[0], x5
	pmull		v0.1q, v0.1d, v3.1d
	pmull		v1.1q, v1.1d, v4.1d
	pmull		v2.1q, v2.1d, v5.1d
	eor		v0.8b, v0.8b, v1.8b
	eor		v0.8b, v0.8b, v2.8b
	mov		x5, v0.d[0]
	eor		x5, x5, x17
	\insn		w0, w6, x5

	mov		in, x9
	cbnz		len, .La\@

	bit\order	w0
	ret
	.endm

	.align		5
SYM_FUNC_START(crc32c_le_arm64_4way)
	crc4way		crc32cx, .L0
SYM_FUNC_END(crc32c_le_arm64_4way)

	.align		5
SYM_FUNC_START(crc32_le_arm64_4way)
	crc4way		crc32x, .L1
SYM_FUNC_END(crc32_le_arm64_4way)

	.align		5
SYM_FUNC_START(crc32_be_arm64_4way)
	crc4way		crc32x, .L1, be
SYM_FUNC_END(crc32_be_arm64_4way)

	.section	.rodata, "a", %progbits
	.align		6
.L0:	.long		0xddc0152b, 0xba4fc28e, 0x493c7d27
	.long		0x0715ce53, 0x9e4addf8, 0xba4fc28e
	.long		0xc96cfdc0, 0x0715ce53, 0xddc0152b
	.long		0xab7aff2a, 0x0d3b6092, 0x9e4addf8
	.long		0x299847d5, 0x878a92a7, 0x39d3b296
	.long		0xb6dd949b, 0xab7aff2a, 0x0715ce53
	.long		0xa60ce07b, 0x83348832, 0x47db8317
	.long		0xd270f1a2, 0xb9e02b86, 0x0d3b6092
	.long		0x65863b64, 0xb6dd949b, 0xc96cfdc0
	.long		0xb3e32c28, 0xbac2fd7b, 0x878a92a7
	.long		0xf285651c, 0xce7f39f4, 0xdaece73e
	.long		0x271d9844, 0xd270f1a2, 0xab7aff2a
	.long		0x6cb08e5c, 0x2b3cac5d, 0x2162d385
	.long		0xcec3662e, 0x1b03397f, 0x83348832
	.long		0x8227bb8a, 0xb3e32c28, 0x299847d5
	.long		0xd7a4825c, 0xdd7e3b0c, 0xb9e02b86
	.long		0xf6076544, 0x10746f3c, 0x18b33a4e
	.long		0x98d8d9cb, 0x271d9844, 0xb6dd949b
	.long		0x57a3d037, 0x93a5f730, 0x78d9ccb7
	.long		0x3771e98f, 0x6b749fb2, 0xbac2fd7b
	.long		0xe0ac139e, 0xcec3662e, 0xa60ce07b
	.long		0x6f345e45, 0xe6fc4e6a, 0xce7f39f4
	.long		0xa2b73df1, 0xb0cd4768, 0x61d82e56
	.long		0x86d8e4d2, 0xd7a4825c, 0xd270f1a2
	.long		0xa90fd27a, 0x0167d312, 0xc619809d
	.long		0xca6ef3ac, 0x26f6a60a, 0x2b3cac5d
	.long		0x4597456a, 0x98d8d9cb, 0x65863b64
	.long		0xc9c8b782, 0x68bce87a, 0x1b03397f
	.long		0x62ec6c6d, 0x6956fc3b, 0xebb883bd
	.long		0x2342001e, 0x3771e98f, 0xb3e32c28
	.long		0xe8b6368b, 0x2178513a, 0x064f7f26
	.long		0x9ef68d35, 0x170076fa, 0xdd7e3b0c
	.long		0x0b0bf8ca, 0x6f345e45, 0xf285651c
	.long		0x02ee03b2, 0xff0dba97, 0x10746f3c
	.long		0x135c83fd, 0xf872e54c, 0xc7a68855
	.long		0x00bcf5f6, 0x86d8e4d2, 0x271d9844
	.long		0x58ca5f00, 0x5bb8f1bc, 0x8e766a0c
	.long		0xded288f8, 0xb3af077a, 0x93a5f730
	.long		0x37170390, 0xca6ef3ac, 0x6cb08e5c
	.long		0xf48642e9, 0xdd66cbbb, 0x6b749fb2
	.long		0xb25b29f2, 0xe9e28eb4, 0x1393e203
	.long		0x45cddf4e, 0xc9c8b782, 0xcec3662e
	.long		0xdfd94fb2, 0x93e106a4, 0x96c515bb
	.long		0x021ac5ef, 0xd813b325, 0xe6fc4e6a
	.long		0x8e1450f7, 0x2342001e, 0x8227bb8a
	.long		0xe0cdcf86, 0x6d9a4957, 0xb0cd4768
	.long		0x613eee91, 0xd2c3ed1a, 0x39c7ff35
	.long		0xbedc6ba1, 0x9ef68d35, 0xd7a4825c
	.long		0x0cd1526a, 0xf2271e60, 0x0ab3844b
	.long		0xd6c3a807, 0x2664fd8b, 0x0167d312
	.long		0x1d31175f, 0x02ee03b2, 0xf6076544
	.long		0x4be7fd90, 0x363bd6b3, 0x26f6a60a
	.long		0x6eeed1c9, 0x5fabe670, 0xa741c1bf
	.long		0xb3a6da94, 0x00bcf5f6, 0x98d8d9cb
	.long		0x2e7d11a7, 0x17f27698, 0x49c3cc9c
	.long		0x889774e1, 0xaa7c7ad5, 0x68bce87a
	.long		0x8a074012, 0xded288f8, 0x57a3d037
	.long		0xbd0bb25f, 0x6d390dec, 0x6956fc3b
	.long		0x3be3c09b, 0x6353c1cc, 0x42d98888
	.long		0x465a4eee, 0xf48642e9, 0x3771e98f
	.long		0x2e5f3c8c, 0xdd35bc8d, 0xb42ae3d9
	.long		0xa52f58ec, 0x9a5ede41, 0x2178513a
	.long		0x47972100, 0x45cddf4e, 0xe0ac139e
	.long		0x359674f7, 0xa51b6135, 0x170076fa

.L1:	.long		0xaf449247, 0x81256527, 0xccaa009e
	.long		0x57c54819, 0x1d9513d7, 0x81256527
	.long		0x3f41287a, 0x57c54819, 0xaf449247
	.long		0xf5e48c85, 0x910eeec1, 0x1d9513d7
	.long		0x1f0c2cdd, 0x9026d5b1, 0xae0b5394
	.long		0x71d54a59, 0xf5e48c85, 0x57c54819
	.long		0x1c63267b, 0xfe807bbd, 0x0cbec0ed
	.long		0xd31343ea, 0xe95c1271, 0x910eeec1
	.long		0xf9d9c7ee, 0x71d54a59, 0x3f41287a
	.long		0x9ee62949, 0xcec97417, 0x9026d5b1
	.long		0xa55d1514, 0xf183c71b, 0xd1df2327
	.long		0x21aa2b26, 0xd31343ea, 0xf5e48c85
	.long		0x9d842b80, 0xeea395c4, 0x3c656ced
	.long		0xd8110ff1, 0xcd669a40, 0xfe807bbd
	.long		0x3f9e9356, 0x9ee62949, 0x1f0c2cdd
	.long		0x1d6708a0, 0x0c30f51d, 0xe95c1271
	.long		0xef82aa68, 0xdb3935ea, 0xb918a347
	.long		0xd14bcc9b, 0x21aa2b26, 0x71d54a59
	.long		0x99cce860, 0x356d209f, 0xff6f2fc2
	.long		0xd8af8e46, 0xc352f6de, 0xcec97417
	.long		0xf1996890, 0xd8110ff1, 0x1c63267b
	.long		0x631bc508, 0xe95c7216, 0xf183c71b
	.long		0x8511c306, 0x8e031a19, 0x9b9bdbd0
	.long		0xdb3839f3, 0x1d6708a0, 0xd31343ea
	.long		0x7a92fffb, 0xf7003835, 0x4470ac44
	.long		0x6ce68f2a, 0x00eba0c8, 0xeea395c4
	.long		0x4caaa263, 0xd14bcc9b, 0xf9d9c7ee
	.long		0xb46f7cff, 0x9a1b53c8, 0xcd669a40
	.long		0x60290934, 0x81b6f443, 0x6d40f445
	.long		0x8e976a7d, 0xd8af8e46, 0x9ee62949
	.long		0xdcf5088a, 0x9dbdc100, 0x145575d5
	.long		0x1753ab84, 0xbbf2f6d6, 0x0c30f51d
	.long		0x255b139e, 0x631bc508, 0xa55d1514
	.long		0xd784eaa8, 0xce26786c, 0xdb3935ea
	.long		0x6d2c864a, 0x8068c345, 0x2586d334
	.long		0x02072e24, 0xdb3839f3, 0x21aa2b26
	.long		0x06689b0a, 0x5efd72f5, 0xe0575528
	.long		0x1e52f5ea, 0x4117915b, 0x356d209f
	.long		0x1d3d1db6, 0x6ce68f2a, 0x9d842b80
	.long		0x3796455c, 0xb8e0e4a8, 0xc352f6de
	.long		0xdf3a4eb3, 0xc55a2330, 0xb84ffa9c
	.long		0x28ae0976, 0xb46f7cff, 0xd8110ff1
	.long		0x9764bc8d, 0xd7e7a22c, 0x712510f0
	.long		0x13a13e18, 0x3e9a43cd, 0xe95c7216
	.long		0xb8ee242e, 0x8e976a7d, 0x3f9e9356
	.long		0x0c540e7b, 0x753c81ff, 0x8e031a19
	.long		0x9924c781, 0xb9220208, 0x3edcde65
	.long		0x3954de39, 0x1753ab84, 0x1d6708a0
	.long		0xf32238b5, 0xbec81497, 0x9e70b943
	.long		0xbbd2cd2c, 0x0925d861, 0xf7003835
	.long		0xcc401304, 0xd784eaa8, 0xef82aa68
	.long		0x4987e684, 0x6044fbb0, 0x00eba0c8
	.long		0x3aa11427, 0x18fe3b4a, 0x87441142
	.long		0x297aad60, 0x02072e24, 0xd14bcc9b
	.long		0xf60c5e51, 0x6ef6f487, 0x5b7fdd0a
	.long		0x632d78c5, 0x3fc33de4, 0x9a1b53c8
	.long		0x25b8822a, 0x1e52f5ea, 0x99cce860
	.long		0xd4fc84bc, 0x1af62fb8, 0x81b6f443
	.long		0x5690aa32, 0xa91fdefb, 0x688a110e
	.long		0x1357a093, 0x3796455c, 0xd8af8e46
	.long		0x798fdd33, 0xaaa18a37, 0x357b9517
	.long		0xc2815395, 0x54d42691, 0x9dbdc100
	.long		0x21cfc0f7, 0x28ae0976, 0xf1996890
	.long		0xa0decef3, 0x7b4aa8b7, 0xbbf2f6d6