Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
torvalds
GitHub Repository: torvalds/linux
 Path: blob/master/arch/loongarch/kernel/kfpu.c
26439 views
1
// SPDX-License-Identifier: GPL-2.0-only

2
/*

3
 * Copyright (C) 2023 Loongson Technology Corporation Limited

4
 */

5


6
#include <linux/cpu.h>

7
#include <linux/export.h>

8
#include <linux/init.h>

9
#include <asm/fpu.h>

10
#include <asm/smp.h>

11


12
static unsigned int euen_mask = CSR_EUEN_FPEN;

13


14
/*

15
 * The critical section between kernel_fpu_begin() and kernel_fpu_end()

16
 * is non-reentrant. It is the caller's responsibility to avoid reentrance.

17
 * See drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c as an example.

18
 */

19
static DEFINE_PER_CPU(bool, in_kernel_fpu);

20
static DEFINE_PER_CPU(unsigned int, euen_current);

21


22
static inline void fpregs_lock(void)

23
{

24
	if (IS_ENABLED(CONFIG_PREEMPT_RT))

25
		preempt_disable();

26
	else

27
		local_bh_disable();

28
}

29


30
static inline void fpregs_unlock(void)

31
{

32
	if (IS_ENABLED(CONFIG_PREEMPT_RT))

33
		preempt_enable();

34
	else

35
		local_bh_enable();

36
}

37


38
void kernel_fpu_begin(void)

39
{

40
	unsigned int *euen_curr;

41


42
	if (!irqs_disabled())

43
		fpregs_lock();

44


45
	WARN_ON(this_cpu_read(in_kernel_fpu));

46


47
	this_cpu_write(in_kernel_fpu, true);

48
	euen_curr = this_cpu_ptr(&euen_current);

49


50
	*euen_curr = csr_xchg32(euen_mask, euen_mask, LOONGARCH_CSR_EUEN);

51


52
#ifdef CONFIG_CPU_HAS_LASX

53
	if (*euen_curr & CSR_EUEN_LASXEN)

54
		_save_lasx(&current->thread.fpu);

55
	else

56
#endif

57
#ifdef CONFIG_CPU_HAS_LSX

58
	if (*euen_curr & CSR_EUEN_LSXEN)

59
		_save_lsx(&current->thread.fpu);

60
	else

61
#endif

62
	if (*euen_curr & CSR_EUEN_FPEN)

63
		_save_fp(&current->thread.fpu);

64


65
	write_fcsr(LOONGARCH_FCSR0, 0);

66
}

67
EXPORT_SYMBOL_GPL(kernel_fpu_begin);

68


69
void kernel_fpu_end(void)

70
{

71
	unsigned int *euen_curr;

72


73
	WARN_ON(!this_cpu_read(in_kernel_fpu));

74


75
	euen_curr = this_cpu_ptr(&euen_current);

76


77
#ifdef CONFIG_CPU_HAS_LASX

78
	if (*euen_curr & CSR_EUEN_LASXEN)

79
		_restore_lasx(&current->thread.fpu);

80
	else

81
#endif

82
#ifdef CONFIG_CPU_HAS_LSX

83
	if (*euen_curr & CSR_EUEN_LSXEN)

84
		_restore_lsx(&current->thread.fpu);

85
	else

86
#endif

87
	if (*euen_curr & CSR_EUEN_FPEN)

88
		_restore_fp(&current->thread.fpu);

89


90
	*euen_curr = csr_xchg32(*euen_curr, euen_mask, LOONGARCH_CSR_EUEN);

91


92
	this_cpu_write(in_kernel_fpu, false);

93


94
	if (!irqs_disabled())

95
		fpregs_unlock();

96
}

97
EXPORT_SYMBOL_GPL(kernel_fpu_end);

98


99
static int __init init_euen_mask(void)

100
{

101
	if (cpu_has_lsx)

102
		euen_mask |= CSR_EUEN_LSXEN;

103


104
	if (cpu_has_lasx)

105
		euen_mask |= CSR_EUEN_LASXEN;

106


107
	return 0;

108
}

109
arch_initcall(init_euen_mask);

110


111