CoCalc -- hypersparc.S

GitHub Repository: torvalds/linux
Path: blob/master/arch/sparc/mm/hypersparc.S
²⁶⁴³⁹ views
1
/* SPDX-License-Identifier: GPL-2.0 */
2
/*
3
 * hypersparc.S: High speed Hypersparc mmu/cache operations.
4
 *
5
 * Copyright (C) 1997 David S. Miller ([email protected])
6
 */
7

8
#include <asm/ptrace.h>
9
#include <asm/psr.h>
10
#include <asm/asm-offsets.h>
11
#include <asm/asi.h>
12
#include <asm/page.h>
13
#include <asm/pgtable.h>
14
#include <asm/pgtsrmmu.h>
15
#include <linux/init.h>
16

17
	.text
18
	.align	4
19

20
	.globl	hypersparc_flush_cache_all, hypersparc_flush_cache_mm
21
	.globl	hypersparc_flush_cache_range, hypersparc_flush_cache_page
22
	.globl	hypersparc_flush_page_to_ram
23
	.globl	hypersparc_flush_page_for_dma, hypersparc_flush_sig_insns
24
	.globl	hypersparc_flush_tlb_all, hypersparc_flush_tlb_mm
25
	.globl	hypersparc_flush_tlb_range, hypersparc_flush_tlb_page
26

27
hypersparc_flush_cache_all:
28
	WINDOW_FLUSH(%g4, %g5)
29
	sethi	%hi(vac_cache_size), %g4
30
	ld	[%g4 + %lo(vac_cache_size)], %g5
31
	sethi	%hi(vac_line_size), %g1
32
	ld	[%g1 + %lo(vac_line_size)], %g2
33
1:	
34
	subcc	%g5, %g2, %g5			! hyper_flush_unconditional_combined
35
	bne	1b
36
	 sta	%g0, [%g5] ASI_M_FLUSH_CTX
37
	retl
38
	 sta	%g0, [%g0] ASI_M_FLUSH_IWHOLE	! hyper_flush_whole_icache
39

40
	/* We expand the window flush to get maximum performance. */
41
hypersparc_flush_cache_mm:
42
#ifndef CONFIG_SMP
43
	ld	[%o0 + AOFF_mm_context], %g1
44
	cmp	%g1, -1
45
	be	hypersparc_flush_cache_mm_out
46
#endif
47
	WINDOW_FLUSH(%g4, %g5)
48

49
	sethi	%hi(vac_line_size), %g1
50
	ld	[%g1 + %lo(vac_line_size)], %o1
51
	sethi	%hi(vac_cache_size), %g2
52
	ld	[%g2 + %lo(vac_cache_size)], %o0
53
	add	%o1, %o1, %g1
54
	add	%o1, %g1, %g2
55
	add	%o1, %g2, %g3
56
	add	%o1, %g3, %g4
57
	add	%o1, %g4, %g5
58
	add	%o1, %g5, %o4
59
	add	%o1, %o4, %o5
60

61
	/* BLAMMO! */
62
1:
63
	subcc	%o0, %o5, %o0				! hyper_flush_cache_user
64
	sta	%g0, [%o0 + %g0] ASI_M_FLUSH_USER
65
	sta	%g0, [%o0 + %o1] ASI_M_FLUSH_USER
66
	sta	%g0, [%o0 + %g1] ASI_M_FLUSH_USER
67
	sta	%g0, [%o0 + %g2] ASI_M_FLUSH_USER
68
	sta	%g0, [%o0 + %g3] ASI_M_FLUSH_USER
69
	sta	%g0, [%o0 + %g4] ASI_M_FLUSH_USER
70
	sta	%g0, [%o0 + %g5] ASI_M_FLUSH_USER
71
	bne	1b
72
	 sta	%g0, [%o0 + %o4] ASI_M_FLUSH_USER
73
hypersparc_flush_cache_mm_out:
74
	retl
75
	 nop
76

77
	/* The things we do for performance... */
78
hypersparc_flush_cache_range:
79
	ld	[%o0 + VMA_VM_MM], %o0
80
#ifndef CONFIG_SMP
81
	ld	[%o0 + AOFF_mm_context], %g1
82
	cmp	%g1, -1
83
	be	hypersparc_flush_cache_range_out
84
#endif
85
	WINDOW_FLUSH(%g4, %g5)
86

87
	sethi	%hi(vac_line_size), %g1
88
	ld	[%g1 + %lo(vac_line_size)], %o4
89
	sethi	%hi(vac_cache_size), %g2
90
	ld	[%g2 + %lo(vac_cache_size)], %o3
91

92
	/* Here comes the fun part... */
93
	add	%o2, (PAGE_SIZE - 1), %o2
94
	andn	%o1, (PAGE_SIZE - 1), %o1
95
	add	%o4, %o4, %o5
96
	andn	%o2, (PAGE_SIZE - 1), %o2
97
	add	%o4, %o5, %g1
98
	sub	%o2, %o1, %g4
99
	add	%o4, %g1, %g2
100
	sll	%o3, 2, %g5
101
	add	%o4, %g2, %g3
102
	cmp	%g4, %g5
103
	add	%o4, %g3, %g4
104
	blu	0f
105
	 add	%o4, %g4, %g5
106
	add	%o4, %g5, %g7
107

108
	/* Flush entire user space, believe it or not this is quicker
109
	 * than page at a time flushings for range > (cache_size<<2).
110
	 */
111
1:
112
	subcc	%o3, %g7, %o3
113
	sta	%g0, [%o3 + %g0] ASI_M_FLUSH_USER
114
	sta	%g0, [%o3 + %o4] ASI_M_FLUSH_USER
115
	sta	%g0, [%o3 + %o5] ASI_M_FLUSH_USER
116
	sta	%g0, [%o3 + %g1] ASI_M_FLUSH_USER
117
	sta	%g0, [%o3 + %g2] ASI_M_FLUSH_USER
118
	sta	%g0, [%o3 + %g3] ASI_M_FLUSH_USER
119
	sta	%g0, [%o3 + %g4] ASI_M_FLUSH_USER
120
	bne	1b
121
	 sta	%g0, [%o3 + %g5] ASI_M_FLUSH_USER
122
	retl
123
	 nop
124

125
	/* Below our threshold, flush one page at a time. */
126
0:
127
	ld	[%o0 + AOFF_mm_context], %o0
128
	mov	SRMMU_CTX_REG, %g7
129
	lda	[%g7] ASI_M_MMUREGS, %o3
130
	sta	%o0, [%g7] ASI_M_MMUREGS
131
	add	%o2, -PAGE_SIZE, %o0
132
1:
133
	or	%o0, 0x400, %g7
134
	lda	[%g7] ASI_M_FLUSH_PROBE, %g7
135
	orcc	%g7, 0, %g0
136
	be,a	3f
137
	 mov	%o0, %o2
138
	add	%o4, %g5, %g7
139
2:
140
	sub	%o2, %g7, %o2
141
	sta	%g0, [%o2 + %g0] ASI_M_FLUSH_PAGE
142
	sta	%g0, [%o2 + %o4] ASI_M_FLUSH_PAGE
143
	sta	%g0, [%o2 + %o5] ASI_M_FLUSH_PAGE
144
	sta	%g0, [%o2 + %g1] ASI_M_FLUSH_PAGE
145
	sta	%g0, [%o2 + %g2] ASI_M_FLUSH_PAGE
146
	sta	%g0, [%o2 + %g3] ASI_M_FLUSH_PAGE
147
	andcc	%o2, 0xffc, %g0
148
	sta	%g0, [%o2 + %g4] ASI_M_FLUSH_PAGE
149
	bne	2b
150
	 sta	%g0, [%o2 + %g5] ASI_M_FLUSH_PAGE
151
3:
152
	cmp	%o2, %o1
153
	bne	1b
154
	 add	%o2, -PAGE_SIZE, %o0
155
	mov	SRMMU_FAULT_STATUS, %g5
156
	lda	[%g5] ASI_M_MMUREGS, %g0
157
	mov	SRMMU_CTX_REG, %g7
158
	sta	%o3, [%g7] ASI_M_MMUREGS
159
hypersparc_flush_cache_range_out:
160
	retl
161
	 nop
162

163
	/* HyperSparc requires a valid mapping where we are about to flush
164
	 * in order to check for a physical tag match during the flush.
165
	 */
166
	/* Verified, my ass... */
167
hypersparc_flush_cache_page:
168
	ld	[%o0 + VMA_VM_MM], %o0
169
	ld	[%o0 + AOFF_mm_context], %g2
170
#ifndef CONFIG_SMP
171
	cmp	%g2, -1
172
	be	hypersparc_flush_cache_page_out
173
#endif
174
	WINDOW_FLUSH(%g4, %g5)
175

176
	sethi	%hi(vac_line_size), %g1
177
	ld	[%g1 + %lo(vac_line_size)], %o4
178
	mov	SRMMU_CTX_REG, %o3
179
	andn	%o1, (PAGE_SIZE - 1), %o1
180
	lda	[%o3] ASI_M_MMUREGS, %o2
181
	sta	%g2, [%o3] ASI_M_MMUREGS
182
	or	%o1, 0x400, %o5
183
	lda	[%o5] ASI_M_FLUSH_PROBE, %g1
184
	orcc	%g0, %g1, %g0
185
	be	2f
186
	 add	%o4, %o4, %o5
187
	sub	%o1, -PAGE_SIZE, %o1
188
	add	%o4, %o5, %g1
189
	add	%o4, %g1, %g2
190
	add	%o4, %g2, %g3
191
	add	%o4, %g3, %g4
192
	add	%o4, %g4, %g5
193
	add	%o4, %g5, %g7
194

195
	/* BLAMMO! */
196
1:
197
	sub	%o1, %g7, %o1
198
	sta	%g0, [%o1 + %g0] ASI_M_FLUSH_PAGE
199
	sta	%g0, [%o1 + %o4] ASI_M_FLUSH_PAGE
200
	sta	%g0, [%o1 + %o5] ASI_M_FLUSH_PAGE
201
	sta	%g0, [%o1 + %g1] ASI_M_FLUSH_PAGE
202
	sta	%g0, [%o1 + %g2] ASI_M_FLUSH_PAGE
203
	sta	%g0, [%o1 + %g3] ASI_M_FLUSH_PAGE
204
	andcc	%o1, 0xffc, %g0
205
	sta	%g0, [%o1 + %g4] ASI_M_FLUSH_PAGE
206
	bne	1b
207
	 sta	%g0, [%o1 + %g5] ASI_M_FLUSH_PAGE
208
2:
209
	mov	SRMMU_FAULT_STATUS, %g7
210
	mov	SRMMU_CTX_REG, %g4
211
	lda	[%g7] ASI_M_MMUREGS, %g0
212
	sta	%o2, [%g4] ASI_M_MMUREGS
213
hypersparc_flush_cache_page_out:
214
	retl
215
	 nop
216

217
hypersparc_flush_sig_insns:
218
	flush	%o1
219
	retl
220
	 flush	%o1 + 4
221

222
	/* HyperSparc is copy-back. */
223
hypersparc_flush_page_to_ram:
224
	sethi	%hi(vac_line_size), %g1
225
	ld	[%g1 + %lo(vac_line_size)], %o4
226
	andn	%o0, (PAGE_SIZE - 1), %o0
227
	add	%o4, %o4, %o5
228
	or	%o0, 0x400, %g7
229
	lda	[%g7] ASI_M_FLUSH_PROBE, %g5
230
	add	%o4, %o5, %g1
231
	orcc	%g5, 0, %g0
232
	be	2f
233
	 add	%o4, %g1, %g2
234
	add	%o4, %g2, %g3
235
	sub	%o0, -PAGE_SIZE, %o0
236
	add	%o4, %g3, %g4
237
	add	%o4, %g4, %g5
238
	add	%o4, %g5, %g7
239

240
	/* BLAMMO! */
241
1:
242
	sub	%o0, %g7, %o0
243
	sta	%g0, [%o0 + %g0] ASI_M_FLUSH_PAGE
244
	sta	%g0, [%o0 + %o4] ASI_M_FLUSH_PAGE
245
	sta	%g0, [%o0 + %o5] ASI_M_FLUSH_PAGE
246
	sta	%g0, [%o0 + %g1] ASI_M_FLUSH_PAGE
247
	sta	%g0, [%o0 + %g2] ASI_M_FLUSH_PAGE
248
	sta	%g0, [%o0 + %g3] ASI_M_FLUSH_PAGE
249
	andcc	%o0, 0xffc, %g0
250
	sta	%g0, [%o0 + %g4] ASI_M_FLUSH_PAGE
251
	bne	1b
252
	 sta	%g0, [%o0 + %g5] ASI_M_FLUSH_PAGE
253
2:
254
	mov	SRMMU_FAULT_STATUS, %g1
255
	retl
256
	 lda	[%g1] ASI_M_MMUREGS, %g0
257

258
	/* HyperSparc is IO cache coherent. */
259
hypersparc_flush_page_for_dma:
260
	retl
261
	 nop
262

263
	/* It was noted that at boot time a TLB flush all in a delay slot
264
	 * can deliver an illegal instruction to the processor if the timing
265
	 * is just right...
266
	 */
267
hypersparc_flush_tlb_all:
268
	mov	0x400, %g1
269
	sta	%g0, [%g1] ASI_M_FLUSH_PROBE
270
	retl
271
	 nop
272

273
hypersparc_flush_tlb_mm:
274
	mov	SRMMU_CTX_REG, %g1
275
	ld	[%o0 + AOFF_mm_context], %o1
276
	lda	[%g1] ASI_M_MMUREGS, %g5
277
#ifndef CONFIG_SMP
278
	cmp	%o1, -1
279
	be	hypersparc_flush_tlb_mm_out
280
#endif
281
	 mov	0x300, %g2
282
	sta	%o1, [%g1] ASI_M_MMUREGS
283
	sta	%g0, [%g2] ASI_M_FLUSH_PROBE
284
hypersparc_flush_tlb_mm_out:
285
	retl
286
	 sta	%g5, [%g1] ASI_M_MMUREGS
287

288
hypersparc_flush_tlb_range:
289
	ld	[%o0 + VMA_VM_MM], %o0
290
	mov	SRMMU_CTX_REG, %g1
291
	ld	[%o0 + AOFF_mm_context], %o3
292
	lda	[%g1] ASI_M_MMUREGS, %g5
293
#ifndef CONFIG_SMP
294
	cmp	%o3, -1
295
	be	hypersparc_flush_tlb_range_out
296
#endif
297
	 sethi	%hi(~((1 << PGDIR_SHIFT) - 1)), %o4
298
	sta	%o3, [%g1] ASI_M_MMUREGS
299
	and	%o1, %o4, %o1
300
	add	%o1, 0x200, %o1
301
	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
302
1:
303
	sub	%o1, %o4, %o1
304
	cmp	%o1, %o2
305
	blu,a	1b
306
	 sta	%g0, [%o1] ASI_M_FLUSH_PROBE
307
hypersparc_flush_tlb_range_out:
308
	retl
309
	 sta	%g5, [%g1] ASI_M_MMUREGS
310

311
hypersparc_flush_tlb_page:
312
	ld	[%o0 + VMA_VM_MM], %o0
313
	mov	SRMMU_CTX_REG, %g1
314
	ld	[%o0 + AOFF_mm_context], %o3
315
	andn	%o1, (PAGE_SIZE - 1), %o1
316
#ifndef CONFIG_SMP
317
	cmp	%o3, -1
318
	be	hypersparc_flush_tlb_page_out
319
#endif
320
	 lda	[%g1] ASI_M_MMUREGS, %g5
321
	sta	%o3, [%g1] ASI_M_MMUREGS
322
	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
323
hypersparc_flush_tlb_page_out:
324
	retl
325
	 sta	%g5, [%g1] ASI_M_MMUREGS
326

327
	__INIT
328
	
329
	/* High speed page clear/copy. */
330
hypersparc_bzero_1page:
331
/* NOTE: This routine has to be shorter than 40insns --jj */
332
	clr	%g1
333
	mov	32, %g2
334
	mov	64, %g3
335
	mov	96, %g4
336
	mov	128, %g5
337
	mov	160, %g7
338
	mov	192, %o2
339
	mov	224, %o3
340
	mov	16, %o1
341
1:
342
	stda	%g0, [%o0 + %g0] ASI_M_BFILL
343
	stda	%g0, [%o0 + %g2] ASI_M_BFILL
344
	stda	%g0, [%o0 + %g3] ASI_M_BFILL
345
	stda	%g0, [%o0 + %g4] ASI_M_BFILL
346
	stda	%g0, [%o0 + %g5] ASI_M_BFILL
347
	stda	%g0, [%o0 + %g7] ASI_M_BFILL
348
	stda	%g0, [%o0 + %o2] ASI_M_BFILL
349
	stda	%g0, [%o0 + %o3] ASI_M_BFILL
350
	subcc	%o1, 1, %o1
351
	bne	1b
352
	 add	%o0, 256, %o0
353

354
	retl
355
	 nop
356

357
hypersparc_copy_1page:
358
/* NOTE: This routine has to be shorter than 70insns --jj */
359
	sub	%o1, %o0, %o2		! difference
360
	mov	16, %g1
361
1:
362
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
363
	add	%o0, 32, %o0
364
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
365
	add	%o0, 32, %o0
366
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
367
	add	%o0, 32, %o0
368
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
369
	add	%o0, 32, %o0
370
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
371
	add	%o0, 32, %o0
372
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
373
	add	%o0, 32, %o0
374
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
375
	add	%o0, 32, %o0
376
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
377
	subcc	%g1, 1, %g1
378
	bne	1b
379
	 add	%o0, 32, %o0
380

381
	retl
382
	 nop
383

384
	.globl	hypersparc_setup_blockops
385
hypersparc_setup_blockops:
386
	sethi	%hi(bzero_1page), %o0
387
	or	%o0, %lo(bzero_1page), %o0
388
	sethi	%hi(hypersparc_bzero_1page), %o1
389
	or	%o1, %lo(hypersparc_bzero_1page), %o1
390
	sethi	%hi(hypersparc_copy_1page), %o2
391
	or	%o2, %lo(hypersparc_copy_1page), %o2
392
	ld	[%o1], %o4
393
1:
394
	add	%o1, 4, %o1
395
	st	%o4, [%o0]
396
	add	%o0, 4, %o0
397
	cmp	%o1, %o2
398
	bne	1b
399
	 ld	[%o1], %o4
400
	sethi	%hi(__copy_1page), %o0
401
	or	%o0, %lo(__copy_1page), %o0
402
	sethi	%hi(hypersparc_setup_blockops), %o2
403
	or	%o2, %lo(hypersparc_setup_blockops), %o2
404
	ld	[%o1], %o4
405
1:
406
	add	%o1, 4, %o1
407
	st	%o4, [%o0]
408
	add	%o0, 4, %o0
409
	cmp	%o1, %o2
410
	bne	1b
411
	 ld	[%o1], %o4
412
	sta	%g0, [%g0] ASI_M_FLUSH_IWHOLE
413
	retl
414
	 nop
415

416
Product

Resources

Company
