CoCalc -- ultra.S

GitHub Repository: torvalds/linux
Path: blob/master/arch/sparc/mm/ultra.S
²⁶⁴²⁴ views
1
/* SPDX-License-Identifier: GPL-2.0 */
2
/*
3
 * ultra.S: Don't expand these all over the place...
4
 *
5
 * Copyright (C) 1997, 2000, 2008 David S. Miller ([email protected])
6
 */
7

8
#include <linux/pgtable.h>
9
#include <asm/asi.h>
10
#include <asm/page.h>
11
#include <asm/spitfire.h>
12
#include <asm/mmu_context.h>
13
#include <asm/mmu.h>
14
#include <asm/pil.h>
15
#include <asm/head.h>
16
#include <asm/thread_info.h>
17
#include <asm/cacheflush.h>
18
#include <asm/hypervisor.h>
19
#include <asm/cpudata.h>
20

21
	/* Basically, most of the Spitfire vs. Cheetah madness
22
	 * has to do with the fact that Cheetah does not support
23
	 * IMMU flushes out of the secondary context.  Someone needs
24
	 * to throw a south lake birthday party for the folks
25
	 * in Microelectronics who refused to fix this shit.
26
	 */
27

28
	/* This file is meant to be read efficiently by the CPU, not humans.
29
	 * Staraj sie tego nikomu nie pierdolnac...
30
	 */
31
	.text
32
	.align		32
33
	.globl		__flush_tlb_mm
34
__flush_tlb_mm:		/* 19 insns */
35
	/* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
36
	ldxa		[%o1] ASI_DMMU, %g2
37
	cmp		%g2, %o0
38
	bne,pn		%icc, __spitfire_flush_tlb_mm_slow
39
	 mov		0x50, %g3
40
	stxa		%g0, [%g3] ASI_DMMU_DEMAP
41
	stxa		%g0, [%g3] ASI_IMMU_DEMAP
42
	sethi		%hi(KERNBASE), %g3
43
	flush		%g3
44
	retl
45
	 nop
46
	nop
47
	nop
48
	nop
49
	nop
50
	nop
51
	nop
52
	nop
53
	nop
54
	nop
55

56
	.align		32
57
	.globl		__flush_tlb_page
58
__flush_tlb_page:	/* 22 insns */
59
	/* %o0 = context, %o1 = vaddr */
60
	rdpr		%pstate, %g7
61
	andn		%g7, PSTATE_IE, %g2
62
	wrpr		%g2, %pstate
63
	mov		SECONDARY_CONTEXT, %o4
64
	ldxa		[%o4] ASI_DMMU, %g2
65
	stxa		%o0, [%o4] ASI_DMMU
66
	andcc		%o1, 1, %g0
67
	andn		%o1, 1, %o3
68
	be,pn		%icc, 1f
69
	 or		%o3, 0x10, %o3
70
	stxa		%g0, [%o3] ASI_IMMU_DEMAP
71
1:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
72
	membar		#Sync
73
	stxa		%g2, [%o4] ASI_DMMU
74
	sethi		%hi(KERNBASE), %o4
75
	flush		%o4
76
	retl
77
	 wrpr		%g7, 0x0, %pstate
78
	nop
79
	nop
80
	nop
81
	nop
82

83
	.align		32
84
	.globl		__flush_tlb_pending
85
__flush_tlb_pending:	/* 27 insns */
86
	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
87
	rdpr		%pstate, %g7
88
	sllx		%o1, 3, %o1
89
	andn		%g7, PSTATE_IE, %g2
90
	wrpr		%g2, %pstate
91
	mov		SECONDARY_CONTEXT, %o4
92
	ldxa		[%o4] ASI_DMMU, %g2
93
	stxa		%o0, [%o4] ASI_DMMU
94
1:	sub		%o1, (1 << 3), %o1
95
	ldx		[%o2 + %o1], %o3
96
	andcc		%o3, 1, %g0
97
	andn		%o3, 1, %o3
98
	be,pn		%icc, 2f
99
	 or		%o3, 0x10, %o3
100
	stxa		%g0, [%o3] ASI_IMMU_DEMAP
101
2:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
102
	membar		#Sync
103
	brnz,pt		%o1, 1b
104
	 nop
105
	stxa		%g2, [%o4] ASI_DMMU
106
	sethi		%hi(KERNBASE), %o4
107
	flush		%o4
108
	retl
109
	 wrpr		%g7, 0x0, %pstate
110
	nop
111
	nop
112
	nop
113
	nop
114

115
	.align		32
116
	.globl		__flush_tlb_kernel_range
117
__flush_tlb_kernel_range:	/* 31 insns */
118
	/* %o0=start, %o1=end */
119
	cmp		%o0, %o1
120
	be,pn		%xcc, 2f
121
	 sub		%o1, %o0, %o3
122
	srlx		%o3, 18, %o4
123
	brnz,pn		%o4, __spitfire_flush_tlb_kernel_range_slow
124
	 sethi		%hi(PAGE_SIZE), %o4
125
	sub		%o3, %o4, %o3
126
	or		%o0, 0x20, %o0		! Nucleus
127
1:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
128
	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
129
	membar		#Sync
130
	brnz,pt		%o3, 1b
131
	 sub		%o3, %o4, %o3
132
2:	sethi		%hi(KERNBASE), %o3
133
	flush		%o3
134
	retl
135
	 nop
136
	nop
137
	nop
138
	nop
139
	nop
140
	nop
141
	nop
142
	nop
143
	nop
144
	nop
145
	nop
146
	nop
147
	nop
148
	nop
149
	nop
150

151
__spitfire_flush_tlb_kernel_range_slow:
152
	mov		63 * 8, %o4
153
1:	ldxa		[%o4] ASI_ITLB_DATA_ACCESS, %o3
154
	andcc		%o3, 0x40, %g0			/* _PAGE_L_4U */
155
	bne,pn		%xcc, 2f
156
	 mov		TLB_TAG_ACCESS, %o3
157
	stxa		%g0, [%o3] ASI_IMMU
158
	stxa		%g0, [%o4] ASI_ITLB_DATA_ACCESS
159
	membar		#Sync
160
2:	ldxa		[%o4] ASI_DTLB_DATA_ACCESS, %o3
161
	andcc		%o3, 0x40, %g0
162
	bne,pn		%xcc, 2f
163
	 mov		TLB_TAG_ACCESS, %o3
164
	stxa		%g0, [%o3] ASI_DMMU
165
	stxa		%g0, [%o4] ASI_DTLB_DATA_ACCESS
166
	membar		#Sync
167
2:	sub		%o4, 8, %o4
168
	brgez,pt	%o4, 1b
169
	 nop
170
	retl
171
	 nop
172

173
__spitfire_flush_tlb_mm_slow:
174
	rdpr		%pstate, %g1
175
	wrpr		%g1, PSTATE_IE, %pstate
176
	stxa		%o0, [%o1] ASI_DMMU
177
	stxa		%g0, [%g3] ASI_DMMU_DEMAP
178
	stxa		%g0, [%g3] ASI_IMMU_DEMAP
179
	flush		%g6
180
	stxa		%g2, [%o1] ASI_DMMU
181
	sethi		%hi(KERNBASE), %o1
182
	flush		%o1
183
	retl
184
	 wrpr		%g1, 0, %pstate
185

186
/*
187
 * The following code flushes one page_size worth.
188
 */
189
	.section .kprobes.text, "ax"
190
	.align		32
191
	.globl		__flush_icache_page
192
__flush_icache_page:	/* %o0 = phys_page */
193
	srlx		%o0, PAGE_SHIFT, %o0
194
	sethi		%hi(PAGE_OFFSET), %g1
195
	sllx		%o0, PAGE_SHIFT, %o0
196
	sethi		%hi(PAGE_SIZE), %g2
197
	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
198
	add		%o0, %g1, %o0
199
1:	subcc		%g2, 32, %g2
200
	bne,pt		%icc, 1b
201
	 flush		%o0 + %g2
202
	retl
203
	 nop
204

205
#ifdef DCACHE_ALIASING_POSSIBLE
206

207
#if (PAGE_SHIFT != 13)
208
#error only page shift of 13 is supported by dcache flush
209
#endif
210

211
#define DTAG_MASK 0x3
212

213
	/* This routine is Spitfire specific so the hardcoded
214
	 * D-cache size and line-size are OK.
215
	 */
216
	.align		64
217
	.globl		__flush_dcache_page
218
__flush_dcache_page:	/* %o0=kaddr, %o1=flush_icache */
219
	sethi		%hi(PAGE_OFFSET), %g1
220
	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
221
	sub		%o0, %g1, %o0			! physical address
222
	srlx		%o0, 11, %o0			! make D-cache TAG
223
	sethi		%hi(1 << 14), %o2		! D-cache size
224
	sub		%o2, (1 << 5), %o2		! D-cache line size
225
1:	ldxa		[%o2] ASI_DCACHE_TAG, %o3	! load D-cache TAG
226
	andcc		%o3, DTAG_MASK, %g0		! Valid?
227
	be,pn		%xcc, 2f			! Nope, branch
228
	 andn		%o3, DTAG_MASK, %o3		! Clear valid bits
229
	cmp		%o3, %o0			! TAG match?
230
	bne,pt		%xcc, 2f			! Nope, branch
231
	 nop
232
	stxa		%g0, [%o2] ASI_DCACHE_TAG	! Invalidate TAG
233
	membar		#Sync
234
2:	brnz,pt		%o2, 1b
235
	 sub		%o2, (1 << 5), %o2		! D-cache line size
236

237
	/* The I-cache does not snoop local stores so we
238
	 * better flush that too when necessary.
239
	 */
240
	brnz,pt		%o1, __flush_icache_page
241
	 sllx		%o0, 11, %o0
242
	retl
243
	 nop
244

245
#endif /* DCACHE_ALIASING_POSSIBLE */
246

247
	.previous
248

249
	/* Cheetah specific versions, patched at boot time. */
250
__cheetah_flush_tlb_mm: /* 19 insns */
251
	rdpr		%pstate, %g7
252
	andn		%g7, PSTATE_IE, %g2
253
	wrpr		%g2, 0x0, %pstate
254
	wrpr		%g0, 1, %tl
255
	mov		PRIMARY_CONTEXT, %o2
256
	mov		0x40, %g3
257
	ldxa		[%o2] ASI_DMMU, %g2
258
	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o1
259
	sllx		%o1, CTX_PGSZ1_NUC_SHIFT, %o1
260
	or		%o0, %o1, %o0	/* Preserve nucleus page size fields */
261
	stxa		%o0, [%o2] ASI_DMMU
262
	stxa		%g0, [%g3] ASI_DMMU_DEMAP
263
	stxa		%g0, [%g3] ASI_IMMU_DEMAP
264
	stxa		%g2, [%o2] ASI_DMMU
265
	sethi		%hi(KERNBASE), %o2
266
	flush		%o2
267
	wrpr		%g0, 0, %tl
268
	retl
269
	 wrpr		%g7, 0x0, %pstate
270

271
__cheetah_flush_tlb_page:	/* 22 insns */
272
	/* %o0 = context, %o1 = vaddr */
273
	rdpr		%pstate, %g7
274
	andn		%g7, PSTATE_IE, %g2
275
	wrpr		%g2, 0x0, %pstate
276
	wrpr		%g0, 1, %tl
277
	mov		PRIMARY_CONTEXT, %o4
278
	ldxa		[%o4] ASI_DMMU, %g2
279
	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
280
	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
281
	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
282
	stxa		%o0, [%o4] ASI_DMMU
283
	andcc		%o1, 1, %g0
284
	be,pn		%icc, 1f
285
	 andn		%o1, 1, %o3
286
	stxa		%g0, [%o3] ASI_IMMU_DEMAP
287
1:	stxa		%g0, [%o3] ASI_DMMU_DEMAP	
288
	membar		#Sync
289
	stxa		%g2, [%o4] ASI_DMMU
290
	sethi		%hi(KERNBASE), %o4
291
	flush		%o4
292
	wrpr		%g0, 0, %tl
293
	retl
294
	 wrpr		%g7, 0x0, %pstate
295

296
__cheetah_flush_tlb_pending:	/* 27 insns */
297
	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
298
	rdpr		%pstate, %g7
299
	sllx		%o1, 3, %o1
300
	andn		%g7, PSTATE_IE, %g2
301
	wrpr		%g2, 0x0, %pstate
302
	wrpr		%g0, 1, %tl
303
	mov		PRIMARY_CONTEXT, %o4
304
	ldxa		[%o4] ASI_DMMU, %g2
305
	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
306
	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
307
	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
308
	stxa		%o0, [%o4] ASI_DMMU
309
1:	sub		%o1, (1 << 3), %o1
310
	ldx		[%o2 + %o1], %o3
311
	andcc		%o3, 1, %g0
312
	be,pn		%icc, 2f
313
	 andn		%o3, 1, %o3
314
	stxa		%g0, [%o3] ASI_IMMU_DEMAP
315
2:	stxa		%g0, [%o3] ASI_DMMU_DEMAP	
316
	membar		#Sync
317
	brnz,pt		%o1, 1b
318
	 nop
319
	stxa		%g2, [%o4] ASI_DMMU
320
	sethi		%hi(KERNBASE), %o4
321
	flush		%o4
322
	wrpr		%g0, 0, %tl
323
	retl
324
	 wrpr		%g7, 0x0, %pstate
325

326
__cheetah_flush_tlb_kernel_range:	/* 31 insns */
327
	/* %o0=start, %o1=end */
328
	cmp		%o0, %o1
329
	be,pn		%xcc, 2f
330
	 sub		%o1, %o0, %o3
331
	srlx		%o3, 18, %o4
332
	brnz,pn		%o4, 3f
333
	 sethi		%hi(PAGE_SIZE), %o4
334
	sub		%o3, %o4, %o3
335
	or		%o0, 0x20, %o0		! Nucleus
336
1:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
337
	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
338
	membar		#Sync
339
	brnz,pt		%o3, 1b
340
	 sub		%o3, %o4, %o3
341
2:	sethi		%hi(KERNBASE), %o3
342
	flush		%o3
343
	retl
344
	 nop
345
3:	mov		0x80, %o4
346
	stxa		%g0, [%o4] ASI_DMMU_DEMAP
347
	membar		#Sync
348
	stxa		%g0, [%o4] ASI_IMMU_DEMAP
349
	membar		#Sync
350
	retl
351
	 nop
352
	nop
353
	nop
354
	nop
355
	nop
356
	nop
357
	nop
358
	nop
359

360
#ifdef DCACHE_ALIASING_POSSIBLE
361
__cheetah_flush_dcache_page: /* 11 insns */
362
	sethi		%hi(PAGE_OFFSET), %g1
363
	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
364
	sub		%o0, %g1, %o0
365
	sethi		%hi(PAGE_SIZE), %o4
366
1:	subcc		%o4, (1 << 5), %o4
367
	stxa		%g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
368
	membar		#Sync
369
	bne,pt		%icc, 1b
370
	 nop
371
	retl		/* I-cache flush never needed on Cheetah, see callers. */
372
	 nop
373
#endif /* DCACHE_ALIASING_POSSIBLE */
374

375
	/* Hypervisor specific versions, patched at boot time.  */
376
__hypervisor_tlb_tl0_error:
377
	save		%sp, -192, %sp
378
	mov		%i0, %o0
379
	call		hypervisor_tlbop_error
380
	 mov		%i1, %o1
381
	ret
382
	 restore
383

384
__hypervisor_flush_tlb_mm: /* 19 insns */
385
	mov		%o0, %o2	/* ARG2: mmu context */
386
	mov		0, %o0		/* ARG0: CPU lists unimplemented */
387
	mov		0, %o1		/* ARG1: CPU lists unimplemented */
388
	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
389
	mov		HV_FAST_MMU_DEMAP_CTX, %o5
390
	ta		HV_FAST_TRAP
391
	brnz,pn		%o0, 1f
392
	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
393
	retl
394
	 nop
395
1:	sethi		%hi(__hypervisor_tlb_tl0_error), %o5
396
	jmpl		%o5 + %lo(__hypervisor_tlb_tl0_error), %g0
397
	 nop
398
	nop
399
	nop
400
	nop
401
	nop
402
	nop
403
	nop
404

405
__hypervisor_flush_tlb_page: /* 22 insns */
406
	/* %o0 = context, %o1 = vaddr */
407
	mov		%o0, %g2
408
	mov		%o1, %o0              /* ARG0: vaddr + IMMU-bit */
409
	mov		%g2, %o1	      /* ARG1: mmu context */
410
	mov		HV_MMU_ALL, %o2	      /* ARG2: flags */
411
	srlx		%o0, PAGE_SHIFT, %o0
412
	sllx		%o0, PAGE_SHIFT, %o0
413
	ta		HV_MMU_UNMAP_ADDR_TRAP
414
	brnz,pn		%o0, 1f
415
	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
416
	retl
417
	 nop
418
1:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
419
	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
420
	 nop
421
	nop
422
	nop
423
	nop
424
	nop
425
	nop
426
	nop
427
	nop
428
	nop
429

430
__hypervisor_flush_tlb_pending: /* 27 insns */
431
	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
432
	sllx		%o1, 3, %g1
433
	mov		%o2, %g2
434
	mov		%o0, %g3
435
1:	sub		%g1, (1 << 3), %g1
436
	ldx		[%g2 + %g1], %o0      /* ARG0: vaddr + IMMU-bit */
437
	mov		%g3, %o1	      /* ARG1: mmu context */
438
	mov		HV_MMU_ALL, %o2	      /* ARG2: flags */
439
	srlx		%o0, PAGE_SHIFT, %o0
440
	sllx		%o0, PAGE_SHIFT, %o0
441
	ta		HV_MMU_UNMAP_ADDR_TRAP
442
	brnz,pn		%o0, 1f
443
	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
444
	brnz,pt		%g1, 1b
445
	 nop
446
	retl
447
	 nop
448
1:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
449
	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
450
	 nop
451
	nop
452
	nop
453
	nop
454
	nop
455
	nop
456
	nop
457
	nop
458
	nop
459

460
__hypervisor_flush_tlb_kernel_range: /* 31 insns */
461
	/* %o0=start, %o1=end */
462
	cmp		%o0, %o1
463
	be,pn		%xcc, 2f
464
	 sub		%o1, %o0, %g2
465
	srlx		%g2, 18, %g3
466
	brnz,pn		%g3, 4f
467
	 mov		%o0, %g1
468
	sethi		%hi(PAGE_SIZE), %g3
469
	sub		%g2, %g3, %g2
470
1:	add		%g1, %g2, %o0	/* ARG0: virtual address */
471
	mov		0, %o1		/* ARG1: mmu context */
472
	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
473
	ta		HV_MMU_UNMAP_ADDR_TRAP
474
	brnz,pn		%o0, 3f
475
	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
476
	brnz,pt		%g2, 1b
477
	 sub		%g2, %g3, %g2
478
2:	retl
479
	 nop
480
3:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
481
	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
482
	 nop
483
4:	mov		0, %o0		/* ARG0: CPU lists unimplemented */
484
	mov		0, %o1		/* ARG1: CPU lists unimplemented */
485
	mov		0, %o2		/* ARG2: mmu context == nucleus */
486
	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
487
	mov		HV_FAST_MMU_DEMAP_CTX, %o5
488
	ta		HV_FAST_TRAP
489
	brnz,pn		%o0, 3b
490
	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
491
	retl
492
	 nop
493

494
#ifdef DCACHE_ALIASING_POSSIBLE
495
	/* XXX Niagara and friends have an 8K cache, so no aliasing is
496
	 * XXX possible, but nothing explicit in the Hypervisor API
497
	 * XXX guarantees this.
498
	 */
499
__hypervisor_flush_dcache_page:	/* 2 insns */
500
	retl
501
	 nop
502
#endif
503

504
tlb_patch_one:
505
1:	lduw		[%o1], %g1
506
	stw		%g1, [%o0]
507
	flush		%o0
508
	subcc		%o2, 1, %o2
509
	add		%o1, 4, %o1
510
	bne,pt		%icc, 1b
511
	 add		%o0, 4, %o0
512
	retl
513
	 nop
514

515
#ifdef CONFIG_SMP
516
	/* These are all called by the slaves of a cross call, at
517
	 * trap level 1, with interrupts fully disabled.
518
	 *
519
	 * Register usage:
520
	 *   %g5	mm->context	(all tlb flushes)
521
	 *   %g1	address arg 1	(tlb page and range flushes)
522
	 *   %g7	address arg 2	(tlb range flush only)
523
	 *
524
	 *   %g6	scratch 1
525
	 *   %g2	scratch 2
526
	 *   %g3	scratch 3
527
	 *   %g4	scratch 4
528
	 */
529
	.align		32
530
	.globl		xcall_flush_tlb_mm
531
xcall_flush_tlb_mm:	/* 24 insns */
532
	mov		PRIMARY_CONTEXT, %g2
533
	ldxa		[%g2] ASI_DMMU, %g3
534
	srlx		%g3, CTX_PGSZ1_NUC_SHIFT, %g4
535
	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
536
	or		%g5, %g4, %g5	/* Preserve nucleus page size fields */
537
	stxa		%g5, [%g2] ASI_DMMU
538
	mov		0x40, %g4
539
	stxa		%g0, [%g4] ASI_DMMU_DEMAP
540
	stxa		%g0, [%g4] ASI_IMMU_DEMAP
541
	stxa		%g3, [%g2] ASI_DMMU
542
	retry
543
	nop
544
	nop
545
	nop
546
	nop
547
	nop
548
	nop
549
	nop
550
	nop
551
	nop
552
	nop
553
	nop
554
	nop
555
	nop
556

557
	.globl		xcall_flush_tlb_page
558
xcall_flush_tlb_page:	/* 20 insns */
559
	/* %g5=context, %g1=vaddr */
560
	mov		PRIMARY_CONTEXT, %g4
561
	ldxa		[%g4] ASI_DMMU, %g2
562
	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %g4
563
	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
564
	or		%g5, %g4, %g5
565
	mov		PRIMARY_CONTEXT, %g4
566
	stxa		%g5, [%g4] ASI_DMMU
567
	andcc		%g1, 0x1, %g0
568
	be,pn		%icc, 2f
569
	 andn		%g1, 0x1, %g5
570
	stxa		%g0, [%g5] ASI_IMMU_DEMAP
571
2:	stxa		%g0, [%g5] ASI_DMMU_DEMAP
572
	membar		#Sync
573
	stxa		%g2, [%g4] ASI_DMMU
574
	retry
575
	nop
576
	nop
577
	nop
578
	nop
579
	nop
580

581
	.globl		xcall_flush_tlb_kernel_range
582
xcall_flush_tlb_kernel_range:	/* 44 insns */
583
	sethi		%hi(PAGE_SIZE - 1), %g2
584
	or		%g2, %lo(PAGE_SIZE - 1), %g2
585
	andn		%g1, %g2, %g1
586
	andn		%g7, %g2, %g7
587
	sub		%g7, %g1, %g3
588
	srlx		%g3, 18, %g2
589
	brnz,pn		%g2, 2f
590
	 sethi		%hi(PAGE_SIZE), %g2
591
	sub		%g3, %g2, %g3
592
	or		%g1, 0x20, %g1		! Nucleus
593
1:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
594
	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
595
	membar		#Sync
596
	brnz,pt		%g3, 1b
597
	 sub		%g3, %g2, %g3
598
	retry
599
2:	mov		63 * 8, %g1
600
1:	ldxa		[%g1] ASI_ITLB_DATA_ACCESS, %g2
601
	andcc		%g2, 0x40, %g0			/* _PAGE_L_4U */
602
	bne,pn		%xcc, 2f
603
	 mov		TLB_TAG_ACCESS, %g2
604
	stxa		%g0, [%g2] ASI_IMMU
605
	stxa		%g0, [%g1] ASI_ITLB_DATA_ACCESS
606
	membar		#Sync
607
2:	ldxa		[%g1] ASI_DTLB_DATA_ACCESS, %g2
608
	andcc		%g2, 0x40, %g0
609
	bne,pn		%xcc, 2f
610
	 mov		TLB_TAG_ACCESS, %g2
611
	stxa		%g0, [%g2] ASI_DMMU
612
	stxa		%g0, [%g1] ASI_DTLB_DATA_ACCESS
613
	membar		#Sync
614
2:	sub		%g1, 8, %g1
615
	brgez,pt	%g1, 1b
616
	 nop
617
	retry
618
	nop
619
	nop
620
	nop
621
	nop
622
	nop
623
	nop
624
	nop
625
	nop
626
	nop
627

628
	/* This runs in a very controlled environment, so we do
629
	 * not need to worry about BH races etc.
630
	 */
631
	.globl		xcall_sync_tick
632
xcall_sync_tick:
633

634
661:	rdpr		%pstate, %g2
635
	wrpr		%g2, PSTATE_IG | PSTATE_AG, %pstate
636
	.section	.sun4v_2insn_patch, "ax"
637
	.word		661b
638
	nop
639
	nop
640
	.previous
641

642
	rdpr		%pil, %g2
643
	wrpr		%g0, PIL_NORMAL_MAX, %pil
644
	sethi		%hi(109f), %g7
645
	b,pt		%xcc, etrap_irq
646
109:	 or		%g7, %lo(109b), %g7
647
#ifdef CONFIG_TRACE_IRQFLAGS
648
	call		trace_hardirqs_off
649
	 nop
650
#endif
651
	call		smp_synchronize_tick_client
652
	 nop
653
	b		rtrap_xcall
654
	 ldx		[%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
655

656
	.globl		xcall_fetch_glob_regs
657
xcall_fetch_glob_regs:
658
	sethi		%hi(global_cpu_snapshot), %g1
659
	or		%g1, %lo(global_cpu_snapshot), %g1
660
	__GET_CPUID(%g2)
661
	sllx		%g2, 6, %g3
662
	add		%g1, %g3, %g1
663
	rdpr		%tstate, %g7
664
	stx		%g7, [%g1 + GR_SNAP_TSTATE]
665
	rdpr		%tpc, %g7
666
	stx		%g7, [%g1 + GR_SNAP_TPC]
667
	rdpr		%tnpc, %g7
668
	stx		%g7, [%g1 + GR_SNAP_TNPC]
669
	stx		%o7, [%g1 + GR_SNAP_O7]
670
	stx		%i7, [%g1 + GR_SNAP_I7]
671
	/* Don't try this at home kids... */
672
	rdpr		%cwp, %g3
673
	sub		%g3, 1, %g7
674
	wrpr		%g7, %cwp
675
	mov		%i7, %g7
676
	wrpr		%g3, %cwp
677
	stx		%g7, [%g1 + GR_SNAP_RPC]
678
	sethi		%hi(trap_block), %g7
679
	or		%g7, %lo(trap_block), %g7
680
	sllx		%g2, TRAP_BLOCK_SZ_SHIFT, %g2
681
	add		%g7, %g2, %g7
682
	ldx		[%g7 + TRAP_PER_CPU_THREAD], %g3
683
	stx		%g3, [%g1 + GR_SNAP_THREAD]
684
	retry
685

686
	.globl		xcall_fetch_glob_pmu
687
xcall_fetch_glob_pmu:
688
	sethi		%hi(global_cpu_snapshot), %g1
689
	or		%g1, %lo(global_cpu_snapshot), %g1
690
	__GET_CPUID(%g2)
691
	sllx		%g2, 6, %g3
692
	add		%g1, %g3, %g1
693
	rd		%pic, %g7
694
	stx		%g7, [%g1 + (4 * 8)]
695
	rd		%pcr, %g7
696
	stx		%g7, [%g1 + (0 * 8)]
697
	retry
698

699
	.globl		xcall_fetch_glob_pmu_n4
700
xcall_fetch_glob_pmu_n4:
701
	sethi		%hi(global_cpu_snapshot), %g1
702
	or		%g1, %lo(global_cpu_snapshot), %g1
703
	__GET_CPUID(%g2)
704
	sllx		%g2, 6, %g3
705
	add		%g1, %g3, %g1
706

707
	ldxa		[%g0] ASI_PIC, %g7
708
	stx		%g7, [%g1 + (4 * 8)]
709
	mov		0x08, %g3
710
	ldxa		[%g3] ASI_PIC, %g7
711
	stx		%g7, [%g1 + (5 * 8)]
712
	mov		0x10, %g3
713
	ldxa		[%g3] ASI_PIC, %g7
714
	stx		%g7, [%g1 + (6 * 8)]
715
	mov		0x18, %g3
716
	ldxa		[%g3] ASI_PIC, %g7
717
	stx		%g7, [%g1 + (7 * 8)]
718

719
	mov		%o0, %g2
720
	mov		%o1, %g3
721
	mov		%o5, %g7
722

723
	mov		HV_FAST_VT_GET_PERFREG, %o5
724
	mov		3, %o0
725
	ta		HV_FAST_TRAP
726
	stx		%o1, [%g1 + (3 * 8)]
727
	mov		HV_FAST_VT_GET_PERFREG, %o5
728
	mov		2, %o0
729
	ta		HV_FAST_TRAP
730
	stx		%o1, [%g1 + (2 * 8)]
731
	mov		HV_FAST_VT_GET_PERFREG, %o5
732
	mov		1, %o0
733
	ta		HV_FAST_TRAP
734
	stx		%o1, [%g1 + (1 * 8)]
735
	mov		HV_FAST_VT_GET_PERFREG, %o5
736
	mov		0, %o0
737
	ta		HV_FAST_TRAP
738
	stx		%o1, [%g1 + (0 * 8)]
739

740
	mov		%g2, %o0
741
	mov		%g3, %o1
742
	mov		%g7, %o5
743

744
	retry
745

746
__cheetah_xcall_flush_tlb_kernel_range:	/* 44 insns */
747
	sethi		%hi(PAGE_SIZE - 1), %g2
748
	or		%g2, %lo(PAGE_SIZE - 1), %g2
749
	andn		%g1, %g2, %g1
750
	andn		%g7, %g2, %g7
751
	sub		%g7, %g1, %g3
752
	srlx		%g3, 18, %g2
753
	brnz,pn		%g2, 2f
754
	 sethi		%hi(PAGE_SIZE), %g2
755
	sub		%g3, %g2, %g3
756
	or		%g1, 0x20, %g1		! Nucleus
757
1:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
758
	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
759
	membar		#Sync
760
	brnz,pt		%g3, 1b
761
	 sub		%g3, %g2, %g3
762
	retry
763
2:	mov		0x80, %g2
764
	stxa		%g0, [%g2] ASI_DMMU_DEMAP
765
	membar		#Sync
766
	stxa		%g0, [%g2] ASI_IMMU_DEMAP
767
	membar		#Sync
768
	retry
769
	nop
770
	nop
771
	nop
772
	nop
773
	nop
774
	nop
775
	nop
776
	nop
777
	nop
778
	nop
779
	nop
780
	nop
781
	nop
782
	nop
783
	nop
784
	nop
785
	nop
786
	nop
787
	nop
788
	nop
789
	nop
790
	nop
791

792
#ifdef DCACHE_ALIASING_POSSIBLE
793
	.align		32
794
	.globl		xcall_flush_dcache_page_cheetah
795
xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
796
	sethi		%hi(PAGE_SIZE), %g3
797
1:	subcc		%g3, (1 << 5), %g3
798
	stxa		%g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
799
	membar		#Sync
800
	bne,pt		%icc, 1b
801
	 nop
802
	retry
803
	nop
804
#endif /* DCACHE_ALIASING_POSSIBLE */
805

806
	.globl		xcall_flush_dcache_page_spitfire
807
xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
808
				     %g7 == kernel page virtual address
809
				     %g5 == (page->mapping != NULL)  */
810
#ifdef DCACHE_ALIASING_POSSIBLE
811
	srlx		%g1, (13 - 2), %g1	! Form tag comparitor
812
	sethi		%hi(L1DCACHE_SIZE), %g3	! D$ size == 16K
813
	sub		%g3, (1 << 5), %g3	! D$ linesize == 32
814
1:	ldxa		[%g3] ASI_DCACHE_TAG, %g2
815
	andcc		%g2, 0x3, %g0
816
	be,pn		%xcc, 2f
817
	 andn		%g2, 0x3, %g2
818
	cmp		%g2, %g1
819

820
	bne,pt		%xcc, 2f
821
	 nop
822
	stxa		%g0, [%g3] ASI_DCACHE_TAG
823
	membar		#Sync
824
2:	cmp		%g3, 0
825
	bne,pt		%xcc, 1b
826
	 sub		%g3, (1 << 5), %g3
827

828
	brz,pn		%g5, 2f
829
#endif /* DCACHE_ALIASING_POSSIBLE */
830
	 sethi		%hi(PAGE_SIZE), %g3
831

832
1:	flush		%g7
833
	subcc		%g3, (1 << 5), %g3
834
	bne,pt		%icc, 1b
835
	 add		%g7, (1 << 5), %g7
836

837
2:	retry
838
	nop
839
	nop
840

841
	/* %g5:	error
842
	 * %g6:	tlb op
843
	 */
844
__hypervisor_tlb_xcall_error:
845
	mov	%g5, %g4
846
	mov	%g6, %g5
847
	ba,pt	%xcc, etrap
848
	 rd	%pc, %g7
849
	mov	%l4, %o0
850
	call	hypervisor_tlbop_error_xcall
851
	 mov	%l5, %o1
852
	ba,a,pt	%xcc, rtrap
853

854
	.globl		__hypervisor_xcall_flush_tlb_mm
855
__hypervisor_xcall_flush_tlb_mm: /* 24 insns */
856
	/* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
857
	mov		%o0, %g2
858
	mov		%o1, %g3
859
	mov		%o2, %g4
860
	mov		%o3, %g1
861
	mov		%o5, %g7
862
	clr		%o0		/* ARG0: CPU lists unimplemented */
863
	clr		%o1		/* ARG1: CPU lists unimplemented */
864
	mov		%g5, %o2	/* ARG2: mmu context */
865
	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
866
	mov		HV_FAST_MMU_DEMAP_CTX, %o5
867
	ta		HV_FAST_TRAP
868
	mov		HV_FAST_MMU_DEMAP_CTX, %g6
869
	brnz,pn		%o0, 1f
870
	 mov		%o0, %g5
871
	mov		%g2, %o0
872
	mov		%g3, %o1
873
	mov		%g4, %o2
874
	mov		%g1, %o3
875
	mov		%g7, %o5
876
	membar		#Sync
877
	retry
878
1:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
879
	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
880
	 nop
881

882
	.globl		__hypervisor_xcall_flush_tlb_page
883
__hypervisor_xcall_flush_tlb_page: /* 20 insns */
884
	/* %g5=ctx, %g1=vaddr */
885
	mov		%o0, %g2
886
	mov		%o1, %g3
887
	mov		%o2, %g4
888
	mov		%g1, %o0	        /* ARG0: virtual address */
889
	mov		%g5, %o1		/* ARG1: mmu context */
890
	mov		HV_MMU_ALL, %o2		/* ARG2: flags */
891
	srlx		%o0, PAGE_SHIFT, %o0
892
	sllx		%o0, PAGE_SHIFT, %o0
893
	ta		HV_MMU_UNMAP_ADDR_TRAP
894
	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
895
	brnz,a,pn	%o0, 1f
896
	 mov		%o0, %g5
897
	mov		%g2, %o0
898
	mov		%g3, %o1
899
	mov		%g4, %o2
900
	membar		#Sync
901
	retry
902
1:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
903
	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
904
	 nop
905

906
	.globl		__hypervisor_xcall_flush_tlb_kernel_range
907
__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
908
	/* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
909
	sethi		%hi(PAGE_SIZE - 1), %g2
910
	or		%g2, %lo(PAGE_SIZE - 1), %g2
911
	andn		%g1, %g2, %g1
912
	andn		%g7, %g2, %g7
913
	sub		%g7, %g1, %g3
914
	srlx		%g3, 18, %g7
915
	add		%g2, 1, %g2
916
	sub		%g3, %g2, %g3
917
	mov		%o0, %g2
918
	mov		%o1, %g4
919
	brnz,pn		%g7, 2f
920
	 mov		%o2, %g7
921
1:	add		%g1, %g3, %o0	/* ARG0: virtual address */
922
	mov		0, %o1		/* ARG1: mmu context */
923
	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
924
	ta		HV_MMU_UNMAP_ADDR_TRAP
925
	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
926
	brnz,pn		%o0, 1f
927
	 mov		%o0, %g5
928
	sethi		%hi(PAGE_SIZE), %o2
929
	brnz,pt		%g3, 1b
930
	 sub		%g3, %o2, %g3
931
5:	mov		%g2, %o0
932
	mov		%g4, %o1
933
	mov		%g7, %o2
934
	membar		#Sync
935
	retry
936
1:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
937
	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
938
	 nop
939
2:	mov		%o3, %g1
940
	mov		%o5, %g3
941
	mov		0, %o0		/* ARG0: CPU lists unimplemented */
942
	mov		0, %o1		/* ARG1: CPU lists unimplemented */
943
	mov		0, %o2		/* ARG2: mmu context == nucleus */
944
	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
945
	mov		HV_FAST_MMU_DEMAP_CTX, %o5
946
	ta		HV_FAST_TRAP
947
	mov		%g1, %o3
948
	brz,pt		%o0, 5b
949
	 mov		%g3, %o5
950
	mov		HV_FAST_MMU_DEMAP_CTX, %g6
951
	ba,pt		%xcc, 1b
952
	 clr		%g5
953

954
	/* These just get rescheduled to PIL vectors. */
955
	.globl		xcall_call_function
956
xcall_call_function:
957
	wr		%g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
958
	retry
959

960
	.globl		xcall_call_function_single
961
xcall_call_function_single:
962
	wr		%g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint
963
	retry
964

965
	.globl		xcall_receive_signal
966
xcall_receive_signal:
967
	wr		%g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
968
	retry
969

970
	.globl		xcall_capture
971
xcall_capture:
972
	wr		%g0, (1 << PIL_SMP_CAPTURE), %set_softint
973
	retry
974

975
#ifdef CONFIG_KGDB
976
	.globl		xcall_kgdb_capture
977
xcall_kgdb_capture:
978
	wr		%g0, (1 << PIL_KGDB_CAPTURE), %set_softint
979
	retry
980
#endif
981

982
#endif /* CONFIG_SMP */
983

984
	.globl		cheetah_patch_cachetlbops
985
cheetah_patch_cachetlbops:
986
	save		%sp, -128, %sp
987

988
	sethi		%hi(__flush_tlb_mm), %o0
989
	or		%o0, %lo(__flush_tlb_mm), %o0
990
	sethi		%hi(__cheetah_flush_tlb_mm), %o1
991
	or		%o1, %lo(__cheetah_flush_tlb_mm), %o1
992
	call		tlb_patch_one
993
	 mov		19, %o2
994

995
	sethi		%hi(__flush_tlb_page), %o0
996
	or		%o0, %lo(__flush_tlb_page), %o0
997
	sethi		%hi(__cheetah_flush_tlb_page), %o1
998
	or		%o1, %lo(__cheetah_flush_tlb_page), %o1
999
	call		tlb_patch_one
1000
	 mov		22, %o2
1001

1002
	sethi		%hi(__flush_tlb_pending), %o0
1003
	or		%o0, %lo(__flush_tlb_pending), %o0
1004
	sethi		%hi(__cheetah_flush_tlb_pending), %o1
1005
	or		%o1, %lo(__cheetah_flush_tlb_pending), %o1
1006
	call		tlb_patch_one
1007
	 mov		27, %o2
1008

1009
	sethi		%hi(__flush_tlb_kernel_range), %o0
1010
	or		%o0, %lo(__flush_tlb_kernel_range), %o0
1011
	sethi		%hi(__cheetah_flush_tlb_kernel_range), %o1
1012
	or		%o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
1013
	call		tlb_patch_one
1014
	 mov		31, %o2
1015

1016
#ifdef DCACHE_ALIASING_POSSIBLE
1017
	sethi		%hi(__flush_dcache_page), %o0
1018
	or		%o0, %lo(__flush_dcache_page), %o0
1019
	sethi		%hi(__cheetah_flush_dcache_page), %o1
1020
	or		%o1, %lo(__cheetah_flush_dcache_page), %o1
1021
	call		tlb_patch_one
1022
	 mov		11, %o2
1023
#endif /* DCACHE_ALIASING_POSSIBLE */
1024

1025
#ifdef CONFIG_SMP
1026
	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
1027
	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
1028
	sethi		%hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
1029
	or		%o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
1030
	call		tlb_patch_one
1031
	 mov		44, %o2
1032
#endif /* CONFIG_SMP */
1033

1034
	ret
1035
	 restore
1036

1037
	.globl		hypervisor_patch_cachetlbops
1038
hypervisor_patch_cachetlbops:
1039
	save		%sp, -128, %sp
1040

1041
	sethi		%hi(__flush_tlb_mm), %o0
1042
	or		%o0, %lo(__flush_tlb_mm), %o0
1043
	sethi		%hi(__hypervisor_flush_tlb_mm), %o1
1044
	or		%o1, %lo(__hypervisor_flush_tlb_mm), %o1
1045
	call		tlb_patch_one
1046
	 mov		19, %o2
1047

1048
	sethi		%hi(__flush_tlb_page), %o0
1049
	or		%o0, %lo(__flush_tlb_page), %o0
1050
	sethi		%hi(__hypervisor_flush_tlb_page), %o1
1051
	or		%o1, %lo(__hypervisor_flush_tlb_page), %o1
1052
	call		tlb_patch_one
1053
	 mov		22, %o2
1054

1055
	sethi		%hi(__flush_tlb_pending), %o0
1056
	or		%o0, %lo(__flush_tlb_pending), %o0
1057
	sethi		%hi(__hypervisor_flush_tlb_pending), %o1
1058
	or		%o1, %lo(__hypervisor_flush_tlb_pending), %o1
1059
	call		tlb_patch_one
1060
	 mov		27, %o2
1061

1062
	sethi		%hi(__flush_tlb_kernel_range), %o0
1063
	or		%o0, %lo(__flush_tlb_kernel_range), %o0
1064
	sethi		%hi(__hypervisor_flush_tlb_kernel_range), %o1
1065
	or		%o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
1066
	call		tlb_patch_one
1067
	 mov		31, %o2
1068

1069
#ifdef DCACHE_ALIASING_POSSIBLE
1070
	sethi		%hi(__flush_dcache_page), %o0
1071
	or		%o0, %lo(__flush_dcache_page), %o0
1072
	sethi		%hi(__hypervisor_flush_dcache_page), %o1
1073
	or		%o1, %lo(__hypervisor_flush_dcache_page), %o1
1074
	call		tlb_patch_one
1075
	 mov		2, %o2
1076
#endif /* DCACHE_ALIASING_POSSIBLE */
1077

1078
#ifdef CONFIG_SMP
1079
	sethi		%hi(xcall_flush_tlb_mm), %o0
1080
	or		%o0, %lo(xcall_flush_tlb_mm), %o0
1081
	sethi		%hi(__hypervisor_xcall_flush_tlb_mm), %o1
1082
	or		%o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
1083
	call		tlb_patch_one
1084
	 mov		24, %o2
1085

1086
	sethi		%hi(xcall_flush_tlb_page), %o0
1087
	or		%o0, %lo(xcall_flush_tlb_page), %o0
1088
	sethi		%hi(__hypervisor_xcall_flush_tlb_page), %o1
1089
	or		%o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
1090
	call		tlb_patch_one
1091
	 mov		20, %o2
1092

1093
	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
1094
	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
1095
	sethi		%hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
1096
	or		%o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
1097
	call		tlb_patch_one
1098
	 mov		44, %o2
1099
#endif /* CONFIG_SMP */
1100

1101
	ret
1102
	 restore
1103

1104
Product

Resources

Company
