Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
torvalds
GitHub Repository: torvalds/linux
 Path: blob/master/arch/x86/lib/clear_page_64.S
51381 views
1
/* SPDX-License-Identifier: GPL-2.0-only */

2
#include <linux/export.h>

3
#include <linux/linkage.h>

4
#include <linux/cfi_types.h>

5
#include <linux/objtool.h>

6
#include <asm/asm.h>

7


8
/*

9
 * Zero page aligned region.

10
 * %rdi	- dest

11
 * %rcx	- length

12
 */

13
SYM_TYPED_FUNC_START(__clear_pages_unrolled)

14
	shrq   $6, %rcx

15
	.p2align 4

16
.Lloop:

17
	decq	%rcx

18
#define PUT(x) movq %rax,x*8(%rdi)

19
	movq %rax,(%rdi)

20
	PUT(1)

21
	PUT(2)

22
	PUT(3)

23
	PUT(4)

24
	PUT(5)

25
	PUT(6)

26
	PUT(7)

27
	leaq	64(%rdi),%rdi

28
	jnz	.Lloop

29
	nop

30
	RET

31
SYM_FUNC_END(__clear_pages_unrolled)

32
EXPORT_SYMBOL_GPL(__clear_pages_unrolled)

33


34
/*

35
 * Default clear user-space.

36
 * Input:

37
 * rdi destination

38
 * rcx count

39
 * rax is zero

40
 *

41
 * Output:

42
 * rcx: uncleared bytes or 0 if successful.

43
 */

44
SYM_FUNC_START(rep_stos_alternative)

45
	ANNOTATE_NOENDBR

46
	cmpq $64,%rcx

47
	jae .Lunrolled

48


49
	cmp $8,%ecx

50
	jae .Lword

51


52
	testl %ecx,%ecx

53
	je .Lexit

54


55
.Lclear_user_tail:

56
0:	movb %al,(%rdi)

57
	inc %rdi

58
	dec %rcx

59
	jnz .Lclear_user_tail

60
.Lexit:

61
	RET

62


63
	_ASM_EXTABLE_UA( 0b, .Lexit)

64


65
.Lword:

66
1:	movq %rax,(%rdi)

67
	addq $8,%rdi

68
	sub $8,%ecx

69
	je .Lexit

70
	cmp $8,%ecx

71
	jae .Lword

72
	jmp .Lclear_user_tail

73


74
	.p2align 4

75
.Lunrolled:

76
10:	movq %rax,(%rdi)

77
11:	movq %rax,8(%rdi)

78
12:	movq %rax,16(%rdi)

79
13:	movq %rax,24(%rdi)

80
14:	movq %rax,32(%rdi)

81
15:	movq %rax,40(%rdi)

82
16:	movq %rax,48(%rdi)

83
17:	movq %rax,56(%rdi)

84
	addq $64,%rdi

85
	subq $64,%rcx

86
	cmpq $64,%rcx

87
	jae .Lunrolled

88
	cmpl $8,%ecx

89
	jae .Lword

90
	testl %ecx,%ecx

91
	jne .Lclear_user_tail

92
	RET

93


94
	/*

95
	 * If we take an exception on any of the

96
	 * word stores, we know that %rcx isn't zero,

97
	 * so we can just go to the tail clearing to

98
	 * get the exact count.

99
	 *

100
	 * The unrolled case might end up clearing

101
	 * some bytes twice. Don't care.

102
	 *

103
	 * We could use the value in %rdi to avoid

104
	 * a second fault on the exact count case,

105
	 * but do we really care? No.

106
	 *

107
	 * Finally, we could try to align %rdi at the

108
	 * top of the unrolling. But unaligned stores

109
	 * just aren't that common or expensive.

110
	 */

111
	_ASM_EXTABLE_UA( 1b, .Lclear_user_tail)

112
	_ASM_EXTABLE_UA(10b, .Lclear_user_tail)

113
	_ASM_EXTABLE_UA(11b, .Lclear_user_tail)

114
	_ASM_EXTABLE_UA(12b, .Lclear_user_tail)

115
	_ASM_EXTABLE_UA(13b, .Lclear_user_tail)

116
	_ASM_EXTABLE_UA(14b, .Lclear_user_tail)

117
	_ASM_EXTABLE_UA(15b, .Lclear_user_tail)

118
	_ASM_EXTABLE_UA(16b, .Lclear_user_tail)

119
	_ASM_EXTABLE_UA(17b, .Lclear_user_tail)

120
SYM_FUNC_END(rep_stos_alternative)

121
EXPORT_SYMBOL(rep_stos_alternative)

122


123