CoCalc -- checksum

GitHub Repository: torvalds/linux
Path: blob/master/arch/powerpc/lib/checksum_32.S
²⁶⁴³⁹ views
1
/* SPDX-License-Identifier: GPL-2.0-or-later */
2
/*
3
 * This file contains assembly-language implementations
4
 * of IP-style 1's complement checksum routines.
5
 *	
6
 *    Copyright (C) 1995-1996 Gary Thomas ([email protected])
7
 *
8
 * Severely hacked about by Paul Mackerras ([email protected]).
9
 */
10

11
#include <linux/export.h>
12
#include <linux/sys.h>
13
#include <asm/processor.h>
14
#include <asm/cache.h>
15
#include <asm/errno.h>
16
#include <asm/ppc_asm.h>
17

18
	.text
19

20
/*
21
 * computes the checksum of a memory block at buff, length len,
22
 * and adds in "sum" (32-bit)
23
 *
24
 * __csum_partial(buff, len, sum)
25
 */
26
_GLOBAL(__csum_partial)
27
	subi	r3,r3,4
28
	srawi.	r6,r4,2		/* Divide len by 4 and also clear carry */
29
	beq	3f		/* if we're doing < 4 bytes */
30
	andi.	r0,r3,2		/* Align buffer to longword boundary */
31
	beq+	1f
32
	lhz	r0,4(r3)	/* do 2 bytes to get aligned */
33
	subi	r4,r4,2
34
	addi	r3,r3,2
35
	srwi.	r6,r4,2		/* # words to do */
36
	adde	r5,r5,r0
37
	beq	3f
38
1:	andi.	r6,r6,3		/* Prepare to handle words 4 by 4 */
39
	beq	21f
40
	mtctr	r6
41
2:	lwzu	r0,4(r3)
42
	adde	r5,r5,r0
43
	bdnz	2b
44
21:	srwi.	r6,r4,4		/* # blocks of 4 words to do */
45
	beq	3f
46
	lwz	r0,4(r3)
47
	mtctr	r6
48
	lwz	r6,8(r3)
49
	adde	r5,r5,r0
50
	lwz	r7,12(r3)
51
	adde	r5,r5,r6
52
	lwzu	r8,16(r3)
53
	adde	r5,r5,r7
54
	bdz	23f
55
22:	lwz	r0,4(r3)
56
	adde	r5,r5,r8
57
	lwz	r6,8(r3)
58
	adde	r5,r5,r0
59
	lwz	r7,12(r3)
60
	adde	r5,r5,r6
61
	lwzu	r8,16(r3)
62
	adde	r5,r5,r7
63
	bdnz	22b
64
23:	adde	r5,r5,r8
65
3:	andi.	r0,r4,2
66
	beq+	4f
67
	lhz	r0,4(r3)
68
	addi	r3,r3,2
69
	adde	r5,r5,r0
70
4:	andi.	r0,r4,1
71
	beq+	5f
72
	lbz	r0,4(r3)
73
	slwi	r0,r0,8		/* Upper byte of word */
74
	adde	r5,r5,r0
75
5:	addze	r3,r5		/* add in final carry */
76
	blr
77
EXPORT_SYMBOL(__csum_partial)
78

79
/*
80
 * Computes the checksum of a memory block at src, length len,
81
 * and adds in 0xffffffff, while copying the block to dst.
82
 * If an access exception occurs it returns zero.
83
 *
84
 * csum_partial_copy_generic(src, dst, len)
85
 */
86
#define CSUM_COPY_16_BYTES_WITHEX(n)	\
87
8 ## n ## 0:			\
88
	lwz	r7,4(r4);	\
89
8 ## n ## 1:			\
90
	lwz	r8,8(r4);	\
91
8 ## n ## 2:			\
92
	lwz	r9,12(r4);	\
93
8 ## n ## 3:			\
94
	lwzu	r10,16(r4);	\
95
8 ## n ## 4:			\
96
	stw	r7,4(r6);	\
97
	adde	r12,r12,r7;	\
98
8 ## n ## 5:			\
99
	stw	r8,8(r6);	\
100
	adde	r12,r12,r8;	\
101
8 ## n ## 6:			\
102
	stw	r9,12(r6);	\
103
	adde	r12,r12,r9;	\
104
8 ## n ## 7:			\
105
	stwu	r10,16(r6);	\
106
	adde	r12,r12,r10
107

108
#define CSUM_COPY_16_BYTES_EXCODE(n)		\
109
	EX_TABLE(8 ## n ## 0b, fault);	\
110
	EX_TABLE(8 ## n ## 1b, fault);	\
111
	EX_TABLE(8 ## n ## 2b, fault);	\
112
	EX_TABLE(8 ## n ## 3b, fault);	\
113
	EX_TABLE(8 ## n ## 4b, fault);	\
114
	EX_TABLE(8 ## n ## 5b, fault);	\
115
	EX_TABLE(8 ## n ## 6b, fault);	\
116
	EX_TABLE(8 ## n ## 7b, fault);
117

118
	.text
119

120
CACHELINE_BYTES = L1_CACHE_BYTES
121
LG_CACHELINE_BYTES = L1_CACHE_SHIFT
122
CACHELINE_MASK = (L1_CACHE_BYTES-1)
123

124
_GLOBAL(csum_partial_copy_generic)
125
	li	r12,-1
126
	addic	r0,r0,0			/* clear carry */
127
	addi	r6,r4,-4
128
	neg	r0,r4
129
	addi	r4,r3,-4
130
	andi.	r0,r0,CACHELINE_MASK	/* # bytes to start of cache line */
131
	crset	4*cr7+eq
132
	beq	58f
133

134
	cmplw	0,r5,r0			/* is this more than total to do? */
135
	blt	63f			/* if not much to do */
136
	rlwinm	r7,r6,3,0x8
137
	rlwnm	r12,r12,r7,0,31	/* odd destination address: rotate one byte */
138
	cmplwi	cr7,r7,0	/* is destination address even ? */
139
	andi.	r8,r0,3			/* get it word-aligned first */
140
	mtctr	r8
141
	beq+	61f
142
	li	r3,0
143
70:	lbz	r9,4(r4)		/* do some bytes */
144
	addi	r4,r4,1
145
	slwi	r3,r3,8
146
	rlwimi	r3,r9,0,24,31
147
71:	stb	r9,4(r6)
148
	addi	r6,r6,1
149
	bdnz	70b
150
	adde	r12,r12,r3
151
61:	subf	r5,r0,r5
152
	srwi.	r0,r0,2
153
	mtctr	r0
154
	beq	58f
155
72:	lwzu	r9,4(r4)		/* do some words */
156
	adde	r12,r12,r9
157
73:	stwu	r9,4(r6)
158
	bdnz	72b
159

160
58:	srwi.	r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
161
	clrlwi	r5,r5,32-LG_CACHELINE_BYTES
162
	li	r11,4
163
	beq	63f
164

165
	/* Here we decide how far ahead to prefetch the source */
166
	li	r3,4
167
	cmpwi	r0,1
168
	li	r7,0
169
	ble	114f
170
	li	r7,1
171
#if MAX_COPY_PREFETCH > 1
172
	/* Heuristically, for large transfers we prefetch
173
	   MAX_COPY_PREFETCH cachelines ahead.  For small transfers
174
	   we prefetch 1 cacheline ahead. */
175
	cmpwi	r0,MAX_COPY_PREFETCH
176
	ble	112f
177
	li	r7,MAX_COPY_PREFETCH
178
112:	mtctr	r7
179
111:	dcbt	r3,r4
180
	addi	r3,r3,CACHELINE_BYTES
181
	bdnz	111b
182
#else
183
	dcbt	r3,r4
184
	addi	r3,r3,CACHELINE_BYTES
185
#endif /* MAX_COPY_PREFETCH > 1 */
186

187
114:	subf	r8,r7,r0
188
	mr	r0,r7
189
	mtctr	r8
190

191
53:	dcbt	r3,r4
192
54:	dcbz	r11,r6
193
/* the main body of the cacheline loop */
194
	CSUM_COPY_16_BYTES_WITHEX(0)
195
#if L1_CACHE_BYTES >= 32
196
	CSUM_COPY_16_BYTES_WITHEX(1)
197
#if L1_CACHE_BYTES >= 64
198
	CSUM_COPY_16_BYTES_WITHEX(2)
199
	CSUM_COPY_16_BYTES_WITHEX(3)
200
#if L1_CACHE_BYTES >= 128
201
	CSUM_COPY_16_BYTES_WITHEX(4)
202
	CSUM_COPY_16_BYTES_WITHEX(5)
203
	CSUM_COPY_16_BYTES_WITHEX(6)
204
	CSUM_COPY_16_BYTES_WITHEX(7)
205
#endif
206
#endif
207
#endif
208
	bdnz	53b
209
	cmpwi	r0,0
210
	li	r3,4
211
	li	r7,0
212
	bne	114b
213

214
63:	srwi.	r0,r5,2
215
	mtctr	r0
216
	beq	64f
217
30:	lwzu	r0,4(r4)
218
	adde	r12,r12,r0
219
31:	stwu	r0,4(r6)
220
	bdnz	30b
221

222
64:	andi.	r0,r5,2
223
	beq+	65f
224
40:	lhz	r0,4(r4)
225
	addi	r4,r4,2
226
41:	sth	r0,4(r6)
227
	adde	r12,r12,r0
228
	addi	r6,r6,2
229
65:	andi.	r0,r5,1
230
	beq+	66f
231
50:	lbz	r0,4(r4)
232
51:	stb	r0,4(r6)
233
	slwi	r0,r0,8
234
	adde	r12,r12,r0
235
66:	addze	r3,r12
236
	beqlr+	cr7
237
	rlwinm	r3,r3,8,0,31	/* odd destination address: rotate one byte */
238
	blr
239

240
fault:
241
	li	r3,0
242
	blr
243

244
	EX_TABLE(70b, fault);
245
	EX_TABLE(71b, fault);
246
	EX_TABLE(72b, fault);
247
	EX_TABLE(73b, fault);
248
	EX_TABLE(54b, fault);
249

250
/*
251
 * this stuff handles faults in the cacheline loop and branches to either
252
 * fault (if in read part) or fault (if in write part)
253
 */
254
	CSUM_COPY_16_BYTES_EXCODE(0)
255
#if L1_CACHE_BYTES >= 32
256
	CSUM_COPY_16_BYTES_EXCODE(1)
257
#if L1_CACHE_BYTES >= 64
258
	CSUM_COPY_16_BYTES_EXCODE(2)
259
	CSUM_COPY_16_BYTES_EXCODE(3)
260
#if L1_CACHE_BYTES >= 128
261
	CSUM_COPY_16_BYTES_EXCODE(4)
262
	CSUM_COPY_16_BYTES_EXCODE(5)
263
	CSUM_COPY_16_BYTES_EXCODE(6)
264
	CSUM_COPY_16_BYTES_EXCODE(7)
265
#endif
266
#endif
267
#endif
268

269
	EX_TABLE(30b, fault);
270
	EX_TABLE(31b, fault);
271
	EX_TABLE(40b, fault);
272
	EX_TABLE(41b, fault);
273
	EX_TABLE(50b, fault);
274
	EX_TABLE(51b, fault);
275

276
EXPORT_SYMBOL(csum_partial_copy_generic)
277

278
/*
279
 * __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
280
 *			   const struct in6_addr *daddr,
281
 *			   __u32 len, __u8 proto, __wsum sum)
282
 */
283

284
_GLOBAL(csum_ipv6_magic)
285
	lwz	r8, 0(r3)
286
	lwz	r9, 4(r3)
287
	addc	r0, r7, r8
288
	lwz	r10, 8(r3)
289
	adde	r0, r0, r9
290
	lwz	r11, 12(r3)
291
	adde	r0, r0, r10
292
	lwz	r8, 0(r4)
293
	adde	r0, r0, r11
294
	lwz	r9, 4(r4)
295
	adde	r0, r0, r8
296
	lwz	r10, 8(r4)
297
	adde	r0, r0, r9
298
	lwz	r11, 12(r4)
299
	adde	r0, r0, r10
300
	add	r5, r5, r6	/* assumption: len + proto doesn't carry */
301
	adde	r0, r0, r11
302
	adde	r0, r0, r5
303
	addze	r0, r0
304
	rotlwi	r3, r0, 16
305
	add	r3, r0, r3
306
	not	r3, r3
307
	rlwinm	r3, r3, 16, 16, 31
308
	blr
309
EXPORT_SYMBOL(csum_ipv6_magic)
310

311
Product

Resources

Company
