Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
torvalds
GitHub Repository: torvalds/linux
 Path: blob/master/arch/arm64/crypto/aes-ce-ccm-core.S
26424 views
1
/* SPDX-License-Identifier: GPL-2.0-only */

2
/*

3
 * aes-ce-ccm-core.S - AES-CCM transform for ARMv8 with Crypto Extensions

4
 *

5
 * Copyright (C) 2013 - 2017 Linaro Ltd.

6
 * Copyright (C) 2024 Google LLC

7
 *

8
 * Author: Ard Biesheuvel <[email protected]>

9
 */

10


11
#include <linux/linkage.h>

12
#include <asm/assembler.h>

13


14
	.text

15
	.arch	armv8-a+crypto

16


17
	.macro	load_round_keys, rk, nr, tmp

18
	sub	w\tmp, \nr, #10

19
	add	\tmp, \rk, w\tmp, sxtw #4

20
	ld1	{v10.4s-v13.4s}, [\rk]

21
	ld1	{v14.4s-v17.4s}, [\tmp], #64

22
	ld1	{v18.4s-v21.4s}, [\tmp], #64

23
	ld1	{v3.4s-v5.4s}, [\tmp]

24
	.endm

25


26
	.macro	dround, va, vb, vk

27
	aese	\va\().16b, \vk\().16b

28
	aesmc	\va\().16b, \va\().16b

29
	aese	\vb\().16b, \vk\().16b

30
	aesmc	\vb\().16b, \vb\().16b

31
	.endm

32


33
	.macro	aes_encrypt, va, vb, nr

34
	tbz	\nr, #2, .L\@

35
	dround	\va, \vb, v10

36
	dround	\va, \vb, v11

37
	tbz	\nr, #1, .L\@

38
	dround	\va, \vb, v12

39
	dround	\va, \vb, v13

40
.L\@:	.irp	v, v14, v15, v16, v17, v18, v19, v20, v21, v3

41
	dround	\va, \vb, \v

42
	.endr

43
	aese	\va\().16b, v4.16b

44
	aese	\vb\().16b, v4.16b

45
	.endm

46


47
	.macro	aes_ccm_do_crypt,enc

48
	load_round_keys	x3, w4, x10

49


50
	ld1	{v0.16b}, [x5]			/* load mac */

51
	cbz	x2, ce_aes_ccm_final

52
	ldr	x8, [x6, #8]			/* load lower ctr */

53
CPU_LE(	rev	x8, x8			)	/* keep swabbed ctr in reg */

54
0:	/* outer loop */

55
	ld1	{v1.8b}, [x6]			/* load upper ctr */

56
	prfm	pldl1strm, [x1]

57
	add	x8, x8, #1

58
	rev	x9, x8

59
	ins	v1.d[1], x9			/* no carry in lower ctr */

60


61
	aes_encrypt	v0, v1, w4

62


63
	subs	w2, w2, #16

64
	bmi	ce_aes_ccm_crypt_tail

65
	ld1	{v2.16b}, [x1], #16		/* load next input block */

66
	.if	\enc == 1

67
	eor	v2.16b, v2.16b, v5.16b		/* final round enc+mac */

68
	eor	v6.16b, v1.16b, v2.16b		/* xor with crypted ctr */

69
	.else

70
	eor	v2.16b, v2.16b, v1.16b		/* xor with crypted ctr */

71
	eor	v6.16b, v2.16b, v5.16b		/* final round enc */

72
	.endif

73
	eor	v0.16b, v0.16b, v2.16b		/* xor mac with pt ^ rk[last] */

74
	st1	{v6.16b}, [x0], #16		/* write output block */

75
	bne	0b

76
CPU_LE(	rev	x8, x8			)

77
	str	x8, [x6, #8]			/* store lsb end of ctr (BE) */

78
	cbnz	x7, ce_aes_ccm_final

79
	st1	{v0.16b}, [x5]			/* store mac */

80
	ret

81
	.endm

82


83
SYM_FUNC_START_LOCAL(ce_aes_ccm_crypt_tail)

84
	eor	v0.16b, v0.16b, v5.16b		/* final round mac */

85
	eor	v1.16b, v1.16b, v5.16b		/* final round enc */

86


87
	add	x1, x1, w2, sxtw		/* rewind the input pointer (w2 < 0) */

88
	add	x0, x0, w2, sxtw		/* rewind the output pointer */

89


90
	adr_l	x8, .Lpermute			/* load permute vectors */

91
	add	x9, x8, w2, sxtw

92
	sub	x8, x8, w2, sxtw

93
	ld1	{v7.16b-v8.16b}, [x9]

94
	ld1	{v9.16b}, [x8]

95


96
	ld1	{v2.16b}, [x1]			/* load a full block of input */

97
	tbl	v1.16b, {v1.16b}, v7.16b	/* move keystream to end of register */

98
	eor	v7.16b, v2.16b, v1.16b		/* encrypt partial input block */

99
	bif	v2.16b, v7.16b, v22.16b		/* select plaintext */

100
	tbx	v7.16b, {v6.16b}, v8.16b	/* insert output from previous iteration */

101
	tbl	v2.16b, {v2.16b}, v9.16b	/* copy plaintext to start of v2 */

102
	eor	v0.16b, v0.16b, v2.16b		/* fold plaintext into mac */

103


104
	st1	{v7.16b}, [x0]			/* store output block */

105
	cbz	x7, 0f

106


107
SYM_INNER_LABEL(ce_aes_ccm_final, SYM_L_LOCAL)

108
	ld1	{v1.16b}, [x7]			/* load 1st ctriv */

109


110
	aes_encrypt	v0, v1, w4

111


112
	/* final round key cancels out */

113
	eor	v0.16b, v0.16b, v1.16b		/* en-/decrypt the mac */

114
0:	st1	{v0.16b}, [x5]			/* store result */

115
	ret

116
SYM_FUNC_END(ce_aes_ccm_crypt_tail)

117


118
	/*

119
	 * void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes,

120
	 * 			   u8 const rk[], u32 rounds, u8 mac[],

121
	 * 			   u8 ctr[], u8 const final_iv[]);

122
	 * void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes,

123
	 * 			   u8 const rk[], u32 rounds, u8 mac[],

124
	 * 			   u8 ctr[], u8 const final_iv[]);

125
	 */

126
SYM_FUNC_START(ce_aes_ccm_encrypt)

127
	movi	v22.16b, #255

128
	aes_ccm_do_crypt	1

129
SYM_FUNC_END(ce_aes_ccm_encrypt)

130


131
SYM_FUNC_START(ce_aes_ccm_decrypt)

132
	movi	v22.16b, #0

133
	aes_ccm_do_crypt	0

134
SYM_FUNC_END(ce_aes_ccm_decrypt)

135


136
	.section ".rodata", "a"

137
	.align	6

138
	.fill	15, 1, 0xff

139
.Lpermute:

140
	.byte	0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7

141
	.byte	0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf

142
	.fill	15, 1, 0xff

143


144