CoCalc -- hypersparc.S

GitHub Repository: awilliam/linux-vfio
Path: blob/master/arch/sparc/mm/hypersparc.S
¹⁰⁸¹⁷ views
1
/*
2
 * hypersparc.S: High speed Hypersparc mmu/cache operations.
3
 *
4
 * Copyright (C) 1997 David S. Miller ([email protected])
5
 */
6

7
#include <asm/ptrace.h>
8
#include <asm/psr.h>
9
#include <asm/asm-offsets.h>
10
#include <asm/asi.h>
11
#include <asm/page.h>
12
#include <asm/pgtsrmmu.h>
13
#include <linux/init.h>
14

15
	.text
16
	.align	4
17

18
	.globl	hypersparc_flush_cache_all, hypersparc_flush_cache_mm
19
	.globl	hypersparc_flush_cache_range, hypersparc_flush_cache_page
20
	.globl	hypersparc_flush_page_to_ram
21
	.globl	hypersparc_flush_page_for_dma, hypersparc_flush_sig_insns
22
	.globl	hypersparc_flush_tlb_all, hypersparc_flush_tlb_mm
23
	.globl	hypersparc_flush_tlb_range, hypersparc_flush_tlb_page
24

25
hypersparc_flush_cache_all:
26
	WINDOW_FLUSH(%g4, %g5)
27
	sethi	%hi(vac_cache_size), %g4
28
	ld	[%g4 + %lo(vac_cache_size)], %g5
29
	sethi	%hi(vac_line_size), %g1
30
	ld	[%g1 + %lo(vac_line_size)], %g2
31
1:	
32
	subcc	%g5, %g2, %g5			! hyper_flush_unconditional_combined
33
	bne	1b
34
	 sta	%g0, [%g5] ASI_M_FLUSH_CTX
35
	retl
36
	 sta	%g0, [%g0] ASI_M_FLUSH_IWHOLE	! hyper_flush_whole_icache
37

38
	/* We expand the window flush to get maximum performance. */
39
hypersparc_flush_cache_mm:
40
#ifndef CONFIG_SMP
41
	ld	[%o0 + AOFF_mm_context], %g1
42
	cmp	%g1, -1
43
	be	hypersparc_flush_cache_mm_out
44
#endif
45
	WINDOW_FLUSH(%g4, %g5)
46

47
	sethi	%hi(vac_line_size), %g1
48
	ld	[%g1 + %lo(vac_line_size)], %o1
49
	sethi	%hi(vac_cache_size), %g2
50
	ld	[%g2 + %lo(vac_cache_size)], %o0
51
	add	%o1, %o1, %g1
52
	add	%o1, %g1, %g2
53
	add	%o1, %g2, %g3
54
	add	%o1, %g3, %g4
55
	add	%o1, %g4, %g5
56
	add	%o1, %g5, %o4
57
	add	%o1, %o4, %o5
58

59
	/* BLAMMO! */
60
1:
61
	subcc	%o0, %o5, %o0				! hyper_flush_cache_user
62
	sta	%g0, [%o0 + %g0] ASI_M_FLUSH_USER
63
	sta	%g0, [%o0 + %o1] ASI_M_FLUSH_USER
64
	sta	%g0, [%o0 + %g1] ASI_M_FLUSH_USER
65
	sta	%g0, [%o0 + %g2] ASI_M_FLUSH_USER
66
	sta	%g0, [%o0 + %g3] ASI_M_FLUSH_USER
67
	sta	%g0, [%o0 + %g4] ASI_M_FLUSH_USER
68
	sta	%g0, [%o0 + %g5] ASI_M_FLUSH_USER
69
	bne	1b
70
	 sta	%g0, [%o0 + %o4] ASI_M_FLUSH_USER
71
hypersparc_flush_cache_mm_out:
72
	retl
73
	 nop
74

75
	/* The things we do for performance... */
76
hypersparc_flush_cache_range:
77
	ld	[%o0 + 0x0], %o0		/* XXX vma->vm_mm, GROSS XXX */
78
#ifndef CONFIG_SMP
79
	ld	[%o0 + AOFF_mm_context], %g1
80
	cmp	%g1, -1
81
	be	hypersparc_flush_cache_range_out
82
#endif
83
	WINDOW_FLUSH(%g4, %g5)
84

85
	sethi	%hi(vac_line_size), %g1
86
	ld	[%g1 + %lo(vac_line_size)], %o4
87
	sethi	%hi(vac_cache_size), %g2
88
	ld	[%g2 + %lo(vac_cache_size)], %o3
89

90
	/* Here comes the fun part... */
91
	add	%o2, (PAGE_SIZE - 1), %o2
92
	andn	%o1, (PAGE_SIZE - 1), %o1
93
	add	%o4, %o4, %o5
94
	andn	%o2, (PAGE_SIZE - 1), %o2
95
	add	%o4, %o5, %g1
96
	sub	%o2, %o1, %g4
97
	add	%o4, %g1, %g2
98
	sll	%o3, 2, %g5
99
	add	%o4, %g2, %g3
100
	cmp	%g4, %g5
101
	add	%o4, %g3, %g4
102
	blu	0f
103
	 add	%o4, %g4, %g5
104
	add	%o4, %g5, %g7
105

106
	/* Flush entire user space, believe it or not this is quicker
107
	 * than page at a time flushings for range > (cache_size<<2).
108
	 */
109
1:
110
	subcc	%o3, %g7, %o3
111
	sta	%g0, [%o3 + %g0] ASI_M_FLUSH_USER
112
	sta	%g0, [%o3 + %o4] ASI_M_FLUSH_USER
113
	sta	%g0, [%o3 + %o5] ASI_M_FLUSH_USER
114
	sta	%g0, [%o3 + %g1] ASI_M_FLUSH_USER
115
	sta	%g0, [%o3 + %g2] ASI_M_FLUSH_USER
116
	sta	%g0, [%o3 + %g3] ASI_M_FLUSH_USER
117
	sta	%g0, [%o3 + %g4] ASI_M_FLUSH_USER
118
	bne	1b
119
	 sta	%g0, [%o3 + %g5] ASI_M_FLUSH_USER
120
	retl
121
	 nop
122

123
	/* Below our threshold, flush one page at a time. */
124
0:
125
	ld	[%o0 + AOFF_mm_context], %o0
126
	mov	SRMMU_CTX_REG, %g7
127
	lda	[%g7] ASI_M_MMUREGS, %o3
128
	sta	%o0, [%g7] ASI_M_MMUREGS
129
	add	%o2, -PAGE_SIZE, %o0
130
1:
131
	or	%o0, 0x400, %g7
132
	lda	[%g7] ASI_M_FLUSH_PROBE, %g7
133
	orcc	%g7, 0, %g0
134
	be,a	3f
135
	 mov	%o0, %o2
136
	add	%o4, %g5, %g7
137
2:
138
	sub	%o2, %g7, %o2
139
	sta	%g0, [%o2 + %g0] ASI_M_FLUSH_PAGE
140
	sta	%g0, [%o2 + %o4] ASI_M_FLUSH_PAGE
141
	sta	%g0, [%o2 + %o5] ASI_M_FLUSH_PAGE
142
	sta	%g0, [%o2 + %g1] ASI_M_FLUSH_PAGE
143
	sta	%g0, [%o2 + %g2] ASI_M_FLUSH_PAGE
144
	sta	%g0, [%o2 + %g3] ASI_M_FLUSH_PAGE
145
	andcc	%o2, 0xffc, %g0
146
	sta	%g0, [%o2 + %g4] ASI_M_FLUSH_PAGE
147
	bne	2b
148
	 sta	%g0, [%o2 + %g5] ASI_M_FLUSH_PAGE
149
3:
150
	cmp	%o2, %o1
151
	bne	1b
152
	 add	%o2, -PAGE_SIZE, %o0
153
	mov	SRMMU_FAULT_STATUS, %g5
154
	lda	[%g5] ASI_M_MMUREGS, %g0
155
	mov	SRMMU_CTX_REG, %g7
156
	sta	%o3, [%g7] ASI_M_MMUREGS
157
hypersparc_flush_cache_range_out:
158
	retl
159
	 nop
160

161
	/* HyperSparc requires a valid mapping where we are about to flush
162
	 * in order to check for a physical tag match during the flush.
163
	 */
164
	/* Verified, my ass... */
165
hypersparc_flush_cache_page:
166
	ld	[%o0 + 0x0], %o0		/* XXX vma->vm_mm, GROSS XXX */
167
	ld	[%o0 + AOFF_mm_context], %g2
168
#ifndef CONFIG_SMP
169
	cmp	%g2, -1
170
	be	hypersparc_flush_cache_page_out
171
#endif
172
	WINDOW_FLUSH(%g4, %g5)
173

174
	sethi	%hi(vac_line_size), %g1
175
	ld	[%g1 + %lo(vac_line_size)], %o4
176
	mov	SRMMU_CTX_REG, %o3
177
	andn	%o1, (PAGE_SIZE - 1), %o1
178
	lda	[%o3] ASI_M_MMUREGS, %o2
179
	sta	%g2, [%o3] ASI_M_MMUREGS
180
	or	%o1, 0x400, %o5
181
	lda	[%o5] ASI_M_FLUSH_PROBE, %g1
182
	orcc	%g0, %g1, %g0
183
	be	2f
184
	 add	%o4, %o4, %o5
185
	sub	%o1, -PAGE_SIZE, %o1
186
	add	%o4, %o5, %g1
187
	add	%o4, %g1, %g2
188
	add	%o4, %g2, %g3
189
	add	%o4, %g3, %g4
190
	add	%o4, %g4, %g5
191
	add	%o4, %g5, %g7
192

193
	/* BLAMMO! */
194
1:
195
	sub	%o1, %g7, %o1
196
	sta	%g0, [%o1 + %g0] ASI_M_FLUSH_PAGE
197
	sta	%g0, [%o1 + %o4] ASI_M_FLUSH_PAGE
198
	sta	%g0, [%o1 + %o5] ASI_M_FLUSH_PAGE
199
	sta	%g0, [%o1 + %g1] ASI_M_FLUSH_PAGE
200
	sta	%g0, [%o1 + %g2] ASI_M_FLUSH_PAGE
201
	sta	%g0, [%o1 + %g3] ASI_M_FLUSH_PAGE
202
	andcc	%o1, 0xffc, %g0
203
	sta	%g0, [%o1 + %g4] ASI_M_FLUSH_PAGE
204
	bne	1b
205
	 sta	%g0, [%o1 + %g5] ASI_M_FLUSH_PAGE
206
2:
207
	mov	SRMMU_FAULT_STATUS, %g7
208
	mov	SRMMU_CTX_REG, %g4
209
	lda	[%g7] ASI_M_MMUREGS, %g0
210
	sta	%o2, [%g4] ASI_M_MMUREGS
211
hypersparc_flush_cache_page_out:
212
	retl
213
	 nop
214

215
hypersparc_flush_sig_insns:
216
	flush	%o1
217
	retl
218
	 flush	%o1 + 4
219

220
	/* HyperSparc is copy-back. */
221
hypersparc_flush_page_to_ram:
222
	sethi	%hi(vac_line_size), %g1
223
	ld	[%g1 + %lo(vac_line_size)], %o4
224
	andn	%o0, (PAGE_SIZE - 1), %o0
225
	add	%o4, %o4, %o5
226
	or	%o0, 0x400, %g7
227
	lda	[%g7] ASI_M_FLUSH_PROBE, %g5
228
	add	%o4, %o5, %g1
229
	orcc	%g5, 0, %g0
230
	be	2f
231
	 add	%o4, %g1, %g2
232
	add	%o4, %g2, %g3
233
	sub	%o0, -PAGE_SIZE, %o0
234
	add	%o4, %g3, %g4
235
	add	%o4, %g4, %g5
236
	add	%o4, %g5, %g7
237

238
	/* BLAMMO! */
239
1:
240
	sub	%o0, %g7, %o0
241
	sta	%g0, [%o0 + %g0] ASI_M_FLUSH_PAGE
242
	sta	%g0, [%o0 + %o4] ASI_M_FLUSH_PAGE
243
	sta	%g0, [%o0 + %o5] ASI_M_FLUSH_PAGE
244
	sta	%g0, [%o0 + %g1] ASI_M_FLUSH_PAGE
245
	sta	%g0, [%o0 + %g2] ASI_M_FLUSH_PAGE
246
	sta	%g0, [%o0 + %g3] ASI_M_FLUSH_PAGE
247
	andcc	%o0, 0xffc, %g0
248
	sta	%g0, [%o0 + %g4] ASI_M_FLUSH_PAGE
249
	bne	1b
250
	 sta	%g0, [%o0 + %g5] ASI_M_FLUSH_PAGE
251
2:
252
	mov	SRMMU_FAULT_STATUS, %g1
253
	retl
254
	 lda	[%g1] ASI_M_MMUREGS, %g0
255

256
	/* HyperSparc is IO cache coherent. */
257
hypersparc_flush_page_for_dma:
258
	retl
259
	 nop
260

261
	/* It was noted that at boot time a TLB flush all in a delay slot
262
	 * can deliver an illegal instruction to the processor if the timing
263
	 * is just right...
264
	 */
265
hypersparc_flush_tlb_all:
266
	mov	0x400, %g1
267
	sta	%g0, [%g1] ASI_M_FLUSH_PROBE
268
	retl
269
	 nop
270

271
hypersparc_flush_tlb_mm:
272
	mov	SRMMU_CTX_REG, %g1
273
	ld	[%o0 + AOFF_mm_context], %o1
274
	lda	[%g1] ASI_M_MMUREGS, %g5
275
#ifndef CONFIG_SMP
276
	cmp	%o1, -1
277
	be	hypersparc_flush_tlb_mm_out
278
#endif
279
	 mov	0x300, %g2
280
	sta	%o1, [%g1] ASI_M_MMUREGS
281
	sta	%g0, [%g2] ASI_M_FLUSH_PROBE
282
hypersparc_flush_tlb_mm_out:
283
	retl
284
	 sta	%g5, [%g1] ASI_M_MMUREGS
285

286
hypersparc_flush_tlb_range:
287
	ld	[%o0 + 0x00], %o0	/* XXX vma->vm_mm GROSS XXX */
288
	mov	SRMMU_CTX_REG, %g1
289
	ld	[%o0 + AOFF_mm_context], %o3
290
	lda	[%g1] ASI_M_MMUREGS, %g5
291
#ifndef CONFIG_SMP
292
	cmp	%o3, -1
293
	be	hypersparc_flush_tlb_range_out
294
#endif
295
	 sethi	%hi(~((1 << SRMMU_PGDIR_SHIFT) - 1)), %o4
296
	sta	%o3, [%g1] ASI_M_MMUREGS
297
	and	%o1, %o4, %o1
298
	add	%o1, 0x200, %o1
299
	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
300
1:
301
	sub	%o1, %o4, %o1
302
	cmp	%o1, %o2
303
	blu,a	1b
304
	 sta	%g0, [%o1] ASI_M_FLUSH_PROBE
305
hypersparc_flush_tlb_range_out:
306
	retl
307
	 sta	%g5, [%g1] ASI_M_MMUREGS
308

309
hypersparc_flush_tlb_page:
310
	ld	[%o0 + 0x00], %o0	/* XXX vma->vm_mm GROSS XXX */
311
	mov	SRMMU_CTX_REG, %g1
312
	ld	[%o0 + AOFF_mm_context], %o3
313
	andn	%o1, (PAGE_SIZE - 1), %o1
314
#ifndef CONFIG_SMP
315
	cmp	%o3, -1
316
	be	hypersparc_flush_tlb_page_out
317
#endif
318
	 lda	[%g1] ASI_M_MMUREGS, %g5
319
	sta	%o3, [%g1] ASI_M_MMUREGS
320
	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
321
hypersparc_flush_tlb_page_out:
322
	retl
323
	 sta	%g5, [%g1] ASI_M_MMUREGS
324

325
	__INIT
326
	
327
	/* High speed page clear/copy. */
328
hypersparc_bzero_1page:
329
/* NOTE: This routine has to be shorter than 40insns --jj */
330
	clr	%g1
331
	mov	32, %g2
332
	mov	64, %g3
333
	mov	96, %g4
334
	mov	128, %g5
335
	mov	160, %g7
336
	mov	192, %o2
337
	mov	224, %o3
338
	mov	16, %o1
339
1:
340
	stda	%g0, [%o0 + %g0] ASI_M_BFILL
341
	stda	%g0, [%o0 + %g2] ASI_M_BFILL
342
	stda	%g0, [%o0 + %g3] ASI_M_BFILL
343
	stda	%g0, [%o0 + %g4] ASI_M_BFILL
344
	stda	%g0, [%o0 + %g5] ASI_M_BFILL
345
	stda	%g0, [%o0 + %g7] ASI_M_BFILL
346
	stda	%g0, [%o0 + %o2] ASI_M_BFILL
347
	stda	%g0, [%o0 + %o3] ASI_M_BFILL
348
	subcc	%o1, 1, %o1
349
	bne	1b
350
	 add	%o0, 256, %o0
351

352
	retl
353
	 nop
354

355
hypersparc_copy_1page:
356
/* NOTE: This routine has to be shorter than 70insns --jj */
357
	sub	%o1, %o0, %o2		! difference
358
	mov	16, %g1
359
1:
360
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
361
	add	%o0, 32, %o0
362
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
363
	add	%o0, 32, %o0
364
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
365
	add	%o0, 32, %o0
366
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
367
	add	%o0, 32, %o0
368
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
369
	add	%o0, 32, %o0
370
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
371
	add	%o0, 32, %o0
372
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
373
	add	%o0, 32, %o0
374
	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
375
	subcc	%g1, 1, %g1
376
	bne	1b
377
	 add	%o0, 32, %o0
378

379
	retl
380
	 nop
381

382
	.globl	hypersparc_setup_blockops
383
hypersparc_setup_blockops:
384
	sethi	%hi(bzero_1page), %o0
385
	or	%o0, %lo(bzero_1page), %o0
386
	sethi	%hi(hypersparc_bzero_1page), %o1
387
	or	%o1, %lo(hypersparc_bzero_1page), %o1
388
	sethi	%hi(hypersparc_copy_1page), %o2
389
	or	%o2, %lo(hypersparc_copy_1page), %o2
390
	ld	[%o1], %o4
391
1:
392
	add	%o1, 4, %o1
393
	st	%o4, [%o0]
394
	add	%o0, 4, %o0
395
	cmp	%o1, %o2
396
	bne	1b
397
	 ld	[%o1], %o4
398
	sethi	%hi(__copy_1page), %o0
399
	or	%o0, %lo(__copy_1page), %o0
400
	sethi	%hi(hypersparc_setup_blockops), %o2
401
	or	%o2, %lo(hypersparc_setup_blockops), %o2
402
	ld	[%o1], %o4
403
1:
404
	add	%o1, 4, %o1
405
	st	%o4, [%o0]
406
	add	%o0, 4, %o0
407
	cmp	%o1, %o2
408
	bne	1b
409
	 ld	[%o1], %o4
410
	sta	%g0, [%g0] ASI_M_FLUSH_IWHOLE
411
	retl
412
	 nop
413

414
Product

Resources

Company
