Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
torvalds
GitHub Repository: torvalds/linux
 Path: blob/master/arch/x86/lib/copy_user_64.S
26442 views
1
/* SPDX-License-Identifier: GPL-2.0-only */

2
/*

3
 * Copyright 2008 Vitaly Mayatskikh <[email protected]>

4
 * Copyright 2002 Andi Kleen, SuSE Labs.

5
 *

6
 * Functions to copy from and to user space.

7
 */

8


9
#include <linux/export.h>

10
#include <linux/linkage.h>

11
#include <linux/cfi_types.h>

12
#include <linux/objtool.h>

13
#include <asm/cpufeatures.h>

14
#include <asm/alternative.h>

15
#include <asm/asm.h>

16


17
/*

18
 * rep_movs_alternative - memory copy with exception handling.

19
 * This version is for CPUs that don't have FSRM (Fast Short Rep Movs)

20
 *

21
 * Input:

22
 * rdi destination

23
 * rsi source

24
 * rcx count

25
 *

26
 * Output:

27
 * rcx uncopied bytes or 0 if successful.

28
 *

29
 * NOTE! The calling convention is very intentionally the same as

30
 * for 'rep movs', so that we can rewrite the function call with

31
 * just a plain 'rep movs' on machines that have FSRM.  But to make

32
 * it simpler for us, we can clobber rsi/rdi and rax freely.

33
 */

34
SYM_FUNC_START(rep_movs_alternative)

35
	ANNOTATE_NOENDBR

36
	cmpq $64,%rcx

37
	jae .Llarge

38


39
	cmp $8,%ecx

40
	jae .Lword

41


42
	testl %ecx,%ecx

43
	je .Lexit

44


45
.Lcopy_user_tail:

46
0:	movb (%rsi),%al

47
1:	movb %al,(%rdi)

48
	inc %rdi

49
	inc %rsi

50
	dec %rcx

51
	jne .Lcopy_user_tail

52
.Lexit:

53
	RET

54


55
	_ASM_EXTABLE_UA( 0b, .Lexit)

56
	_ASM_EXTABLE_UA( 1b, .Lexit)

57


58
	.p2align 4

59
.Lword:

60
2:	movq (%rsi),%rax

61
3:	movq %rax,(%rdi)

62
	addq $8,%rsi

63
	addq $8,%rdi

64
	sub $8,%ecx

65
	je .Lexit

66
	cmp $8,%ecx

67
	jae .Lword

68
	jmp .Lcopy_user_tail

69


70
	_ASM_EXTABLE_UA( 2b, .Lcopy_user_tail)

71
	_ASM_EXTABLE_UA( 3b, .Lcopy_user_tail)

72


73
.Llarge:

74
0:	ALTERNATIVE "jmp .Llarge_movsq", "rep movsb", X86_FEATURE_ERMS

75
1:	RET

76


77
	_ASM_EXTABLE_UA( 0b, 1b)

78


79
.Llarge_movsq:

80
	/* Do the first possibly unaligned word */

81
0:	movq (%rsi),%rax

82
1:	movq %rax,(%rdi)

83


84
	_ASM_EXTABLE_UA( 0b, .Lcopy_user_tail)

85
	_ASM_EXTABLE_UA( 1b, .Lcopy_user_tail)

86


87
	/* What would be the offset to the aligned destination? */

88
	leaq 8(%rdi),%rax

89
	andq $-8,%rax

90
	subq %rdi,%rax

91


92
	/* .. and update pointers and count to match */

93
	addq %rax,%rdi

94
	addq %rax,%rsi

95
	subq %rax,%rcx

96


97
	/* make %rcx contain the number of words, %rax the remainder */

98
	movq %rcx,%rax

99
	shrq $3,%rcx

100
	andl $7,%eax

101
0:	rep movsq

102
	movl %eax,%ecx

103
	testl %ecx,%ecx

104
	jne .Lcopy_user_tail

105
	RET

106


107
1:	leaq (%rax,%rcx,8),%rcx

108
	jmp .Lcopy_user_tail

109


110
	_ASM_EXTABLE_UA( 0b, 1b)

111
SYM_FUNC_END(rep_movs_alternative)

112
EXPORT_SYMBOL(rep_movs_alternative)

113


114