Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
torvalds
GitHub Repository: torvalds/linux
 Path: blob/master/lib/crypto/x86/blake2s-glue.c
26292 views
1
// SPDX-License-Identifier: GPL-2.0 OR MIT

2
/*

3
 * Copyright (C) 2015-2019 Jason A. Donenfeld <[email protected]>. All Rights Reserved.

4
 */

5


6
#include <asm/cpufeature.h>

7
#include <asm/fpu/api.h>

8
#include <asm/processor.h>

9
#include <asm/simd.h>

10
#include <crypto/internal/blake2s.h>

11
#include <linux/init.h>

12
#include <linux/jump_label.h>

13
#include <linux/kernel.h>

14
#include <linux/sizes.h>

15


16
asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state,

17
				       const u8 *block, const size_t nblocks,

18
				       const u32 inc);

19
asmlinkage void blake2s_compress_avx512(struct blake2s_state *state,

20
					const u8 *block, const size_t nblocks,

21
					const u32 inc);

22


23
static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_ssse3);

24
static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_avx512);

25


26
void blake2s_compress(struct blake2s_state *state, const u8 *block,

27
		      size_t nblocks, const u32 inc)

28
{

29
	/* SIMD disables preemption, so relax after processing each page. */

30
	BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8);

31


32
	if (!static_branch_likely(&blake2s_use_ssse3) || !may_use_simd()) {

33
		blake2s_compress_generic(state, block, nblocks, inc);

34
		return;

35
	}

36


37
	do {

38
		const size_t blocks = min_t(size_t, nblocks,

39
					    SZ_4K / BLAKE2S_BLOCK_SIZE);

40


41
		kernel_fpu_begin();

42
		if (static_branch_likely(&blake2s_use_avx512))

43
			blake2s_compress_avx512(state, block, blocks, inc);

44
		else

45
			blake2s_compress_ssse3(state, block, blocks, inc);

46
		kernel_fpu_end();

47


48
		nblocks -= blocks;

49
		block += blocks * BLAKE2S_BLOCK_SIZE;

50
	} while (nblocks);

51
}

52
EXPORT_SYMBOL(blake2s_compress);

53


54
static int __init blake2s_mod_init(void)

55
{

56
	if (boot_cpu_has(X86_FEATURE_SSSE3))

57
		static_branch_enable(&blake2s_use_ssse3);

58


59
	if (boot_cpu_has(X86_FEATURE_AVX) &&

60
	    boot_cpu_has(X86_FEATURE_AVX2) &&

61
	    boot_cpu_has(X86_FEATURE_AVX512F) &&

62
	    boot_cpu_has(X86_FEATURE_AVX512VL) &&

63
	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |

64
			      XFEATURE_MASK_AVX512, NULL))

65
		static_branch_enable(&blake2s_use_avx512);

66


67
	return 0;

68
}

69


70
subsys_initcall(blake2s_mod_init);

71


72