crc32.h

// SPDX-License-Identifier: GPL-2.0
/*
 * CRC-32 implemented with the z/Architecture Vector Extension Facility.
 *
 * Copyright IBM Corp. 2015
 * Author(s): Hendrik Brueckner <[email protected]>
 */

#include <linux/cpufeature.h>
#include <asm/fpu.h>
#include "crc32-vx.h"

#define VX_MIN_LEN		64
#define VX_ALIGNMENT		16L
#define VX_ALIGN_MASK		(VX_ALIGNMENT - 1)

/*
 * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
 *
 * Creates a function to perform a particular CRC-32 computation. Depending
 * on the message buffer, the hardware-accelerated or software implementation
 * is used.   Note that the message buffer is aligned to improve fetch
 * operations of VECTOR LOAD MULTIPLE instructions.
 */
#define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw)		    \
	static inline u32 ___fname(u32 crc, const u8 *data, size_t datalen) \
	{								    \
		unsigned long prealign, aligned, remaining;		    \
		DECLARE_KERNEL_FPU_ONSTACK16(vxstate);			    \
									    \
		if (datalen < VX_MIN_LEN + VX_ALIGN_MASK || !cpu_has_vx())  \
			return ___crc32_sw(crc, data, datalen);		    \
									    \
		if ((unsigned long)data & VX_ALIGN_MASK) {		    \
			prealign = VX_ALIGNMENT -			    \
				  ((unsigned long)data & VX_ALIGN_MASK);    \
			datalen -= prealign;				    \
			crc = ___crc32_sw(crc, data, prealign);		    \
			data = (void *)((unsigned long)data + prealign);    \
		}							    \
									    \
		aligned = datalen & ~VX_ALIGN_MASK;			    \
		remaining = datalen & VX_ALIGN_MASK;			    \
									    \
		kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW);		    \
		crc = ___crc32_vx(crc, data, aligned);			    \
		kernel_fpu_end(&vxstate, KERNEL_VXR_LOW);		    \
									    \
		if (remaining)						    \
			crc = ___crc32_sw(crc, data + aligned, remaining);  \
									    \
		return crc;						    \
	}

DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base)
DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base)
DEFINE_CRC32_VX(crc32c_arch, crc32c_le_vgfm_16, crc32c_base)

static inline u32 crc32_optimizations_arch(void)
{
	if (cpu_has_vx()) {
		return CRC32_LE_OPTIMIZATION |
		       CRC32_BE_OPTIMIZATION |
		       CRC32C_OPTIMIZATION;
	}
	return 0;
}