cache.S

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * This file contains low-level cache management functions
 * used for sleep and CPU speed changes on Apple machines.
 * (In fact the only thing that is Apple-specific is that we assume
 * that we can read from ROM at physical address 0xfff00000.)
 *
 *    Copyright (C) 2004 Paul Mackerras ([email protected]) and
 *                       Benjamin Herrenschmidt ([email protected])
 */

#include <asm/processor.h>
#include <asm/ppc_asm.h>
#include <asm/cputable.h>
#include <asm/feature-fixups.h>

/*
 * Flush and disable all data caches (dL1, L2, L3). This is used
 * when going to sleep, when doing a PMU based cpufreq transition,
 * or when "offlining" a CPU on SMP machines. This code is over
 * paranoid, but I've had enough issues with various CPU revs and
 * bugs that I decided it was worth being over cautious
 */

_GLOBAL(flush_disable_caches)
#ifndef CONFIG_PPC_BOOK3S_32
	blr
#else
BEGIN_FTR_SECTION
	b	flush_disable_745x
END_FTR_SECTION_IFSET(CPU_FTR_SPEC7450)
BEGIN_FTR_SECTION
	b	flush_disable_75x
END_FTR_SECTION_IFSET(CPU_FTR_L2CR)
	b	__flush_disable_L1

/* This is the code for G3 and 74[01]0 */
flush_disable_75x:
	mflr	r10

	/* Turn off EE and DR in MSR */
	mfmsr	r11
	rlwinm	r0,r11,0,~MSR_EE
	rlwinm	r0,r0,0,~MSR_DR
	sync
	mtmsr	r0
	isync

	/* Stop DST streams */
BEGIN_FTR_SECTION
	PPC_DSSALL
	sync
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)

	/* Stop DPM */
	mfspr	r8,SPRN_HID0		/* Save SPRN_HID0 in r8 */
	rlwinm	r4,r8,0,12,10		/* Turn off HID0[DPM] */
	sync
	mtspr	SPRN_HID0,r4		/* Disable DPM */
	sync

	/* Disp-flush L1. We have a weird problem here that I never
	 * totally figured out. On 750FX, using the ROM for the flush
	 * results in a non-working flush. We use that workaround for
	 * now until I finally understand what's going on. --BenH
	 */

	/* ROM base by default */
	lis	r4,0xfff0
	mfpvr	r3
	srwi	r3,r3,16
	cmplwi	cr0,r3,0x7000
	bne+	1f
	/* RAM base on 750FX */
	li	r4,0
1:	li	r4,0x4000
	mtctr	r4
1:	lwz	r0,0(r4)
	addi	r4,r4,32
	bdnz	1b
	sync
	isync

	/* Disable / invalidate / enable L1 data */
	mfspr	r3,SPRN_HID0
	rlwinm	r3,r3,0,~(HID0_DCE | HID0_ICE)
	mtspr	SPRN_HID0,r3
	sync
	isync
	ori	r3,r3,(HID0_DCE|HID0_DCI|HID0_ICE|HID0_ICFI)
	sync
	isync
	mtspr	SPRN_HID0,r3
	xori	r3,r3,(HID0_DCI|HID0_ICFI)
	mtspr	SPRN_HID0,r3
	sync

	/* Get the current enable bit of the L2CR into r4 */
	mfspr	r5,SPRN_L2CR
	/* Set to data-only (pre-745x bit) */
	oris	r3,r5,L2CR_L2DO@h
	b	2f
	/* When disabling L2, code must be in L1 */
	.balign 32
1:	mtspr	SPRN_L2CR,r3
3:	sync
	isync
	b	1f
2:	b	3f
3:	sync
	isync
	b	1b
1:	/* disp-flush L2. The interesting thing here is that the L2 can be
	 * up to 2Mb ... so using the ROM, we'll end up wrapping back to memory
	 * but that is probbaly fine. We disp-flush over 4Mb to be safe
	 */
	lis	r4,2
	mtctr	r4
	lis	r4,0xfff0
1:	lwz	r0,0(r4)
	addi	r4,r4,32
	bdnz	1b
	sync
	isync
	lis	r4,2
	mtctr	r4
	lis	r4,0xfff0
1:	dcbf	0,r4
	addi	r4,r4,32
	bdnz	1b
	sync
	isync

	/* now disable L2 */
	rlwinm	r5,r5,0,~L2CR_L2E
	b	2f
	/* When disabling L2, code must be in L1 */
	.balign 32
1:	mtspr	SPRN_L2CR,r5
3:	sync
	isync
	b	1f
2:	b	3f
3:	sync
	isync
	b	1b
1:	sync
	isync
	/* Invalidate L2. This is pre-745x, we clear the L2I bit ourselves */
	oris	r4,r5,L2CR_L2I@h
	mtspr	SPRN_L2CR,r4
	sync
	isync

	/* Wait for the invalidation to complete */
1:	mfspr	r3,SPRN_L2CR
	rlwinm.	r0,r3,0,31,31
	bne	1b

	/* Clear L2I */
	xoris	r4,r4,L2CR_L2I@h
	sync
	mtspr	SPRN_L2CR,r4
	sync

	/* now disable the L1 data cache */
	mfspr	r0,SPRN_HID0
	rlwinm	r0,r0,0,~(HID0_DCE|HID0_ICE)
	mtspr	SPRN_HID0,r0
	sync
	isync

	/* Restore HID0[DPM] to whatever it was before */
	sync
	mfspr	r0,SPRN_HID0
	rlwimi	r0,r8,0,11,11		/* Turn back HID0[DPM] */
	mtspr	SPRN_HID0,r0
	sync

	/* restore DR and EE */
	sync
	mtmsr	r11
	isync

	mtlr	r10
	blr
_ASM_NOKPROBE_SYMBOL(flush_disable_75x)

/* This code is for 745x processors */
flush_disable_745x:
	/* Turn off EE and DR in MSR */
	mfmsr	r11
	rlwinm	r0,r11,0,~MSR_EE
	rlwinm	r0,r0,0,~MSR_DR
	sync
	mtmsr	r0
	isync

	/* Stop prefetch streams */
	PPC_DSSALL
	sync

	/* Disable L2 prefetching */
	mfspr	r0,SPRN_MSSCR0
	rlwinm	r0,r0,0,0,29
	mtspr	SPRN_MSSCR0,r0
	sync
	isync
	lis	r4,0
	dcbf	0,r4
	dcbf	0,r4
	dcbf	0,r4
	dcbf	0,r4
	dcbf	0,r4
	dcbf	0,r4
	dcbf	0,r4
	dcbf	0,r4

	/* Due to a bug with the HW flush on some CPU revs, we occasionally
	 * experience data corruption. I'm adding a displacement flush along
	 * with a dcbf loop over a few Mb to "help". The problem isn't totally
	 * fixed by this in theory, but at least, in practice, I couldn't reproduce
	 * it even with a big hammer...
	 */

        lis     r4,0x0002
        mtctr   r4
 	li      r4,0
1:
        lwz     r0,0(r4)
        addi    r4,r4,32                /* Go to start of next cache line */
        bdnz    1b
        isync

        /* Now, flush the first 4MB of memory */
        lis     r4,0x0002
        mtctr   r4
	li      r4,0
        sync
1:
        dcbf    0,r4
        addi    r4,r4,32                /* Go to start of next cache line */
        bdnz    1b

	/* Flush and disable the L1 data cache */
	mfspr	r6,SPRN_LDSTCR
	lis	r3,0xfff0	/* read from ROM for displacement flush */
	li	r4,0xfe		/* start with only way 0 unlocked */
	li	r5,128		/* 128 lines in each way */
1:	mtctr	r5
	rlwimi	r6,r4,0,24,31
	mtspr	SPRN_LDSTCR,r6
	sync
	isync
2:	lwz	r0,0(r3)	/* touch each cache line */
	addi	r3,r3,32
	bdnz	2b
	rlwinm	r4,r4,1,24,30	/* move on to the next way */
	ori	r4,r4,1
	cmpwi	r4,0xff		/* all done? */
	bne	1b
	/* now unlock the L1 data cache */
	li	r4,0
	rlwimi	r6,r4,0,24,31
	sync
	mtspr	SPRN_LDSTCR,r6
	sync
	isync

	/* Flush the L2 cache using the hardware assist */
	mfspr	r3,SPRN_L2CR
	cmpwi	r3,0		/* check if it is enabled first */
	bge	4f
	oris	r0,r3,(L2CR_L2IO_745x|L2CR_L2DO_745x)@h
	b	2f
	/* When disabling/locking L2, code must be in L1 */
	.balign 32
1:	mtspr	SPRN_L2CR,r0	/* lock the L2 cache */
3:	sync
	isync
	b	1f
2:	b	3f
3:	sync
	isync
	b	1b
1:	sync
	isync
	ori	r0,r3,L2CR_L2HWF_745x
	sync
	mtspr	SPRN_L2CR,r0	/* set the hardware flush bit */
3:	mfspr	r0,SPRN_L2CR	/* wait for it to go to 0 */
	andi.	r0,r0,L2CR_L2HWF_745x
	bne	3b
	sync
	rlwinm	r3,r3,0,~L2CR_L2E
	b	2f
	/* When disabling L2, code must be in L1 */
	.balign 32
1:	mtspr	SPRN_L2CR,r3	/* disable the L2 cache */
3:	sync
	isync
	b	1f
2:	b	3f
3:	sync
	isync
	b	1b
1:	sync
	isync
	oris	r4,r3,L2CR_L2I@h
	mtspr	SPRN_L2CR,r4
	sync
	isync
1:	mfspr	r4,SPRN_L2CR
	andis.	r0,r4,L2CR_L2I@h
	bne	1b
	sync

BEGIN_FTR_SECTION
	/* Flush the L3 cache using the hardware assist */
4:	mfspr	r3,SPRN_L3CR
	cmpwi	r3,0		/* check if it is enabled */
	bge	6f
	oris	r0,r3,L3CR_L3IO@h
	ori	r0,r0,L3CR_L3DO
	sync
	mtspr	SPRN_L3CR,r0	/* lock the L3 cache */
	sync
	isync
	ori	r0,r0,L3CR_L3HWF
	sync
	mtspr	SPRN_L3CR,r0	/* set the hardware flush bit */
5:	mfspr	r0,SPRN_L3CR	/* wait for it to go to zero */
	andi.	r0,r0,L3CR_L3HWF
	bne	5b
	rlwinm	r3,r3,0,~L3CR_L3E
	sync
	mtspr	SPRN_L3CR,r3	/* disable the L3 cache */
	sync
	ori	r4,r3,L3CR_L3I
	mtspr	SPRN_L3CR,r4
1:	mfspr	r4,SPRN_L3CR
	andi.	r0,r4,L3CR_L3I
	bne	1b
	sync
END_FTR_SECTION_IFSET(CPU_FTR_L3CR)

6:	mfspr	r0,SPRN_HID0	/* now disable the L1 data cache */
	rlwinm	r0,r0,0,~HID0_DCE
	mtspr	SPRN_HID0,r0
	sync
	isync
	mtmsr	r11		/* restore DR and EE */
	isync
	blr
_ASM_NOKPROBE_SYMBOL(flush_disable_745x)
#endif	/* CONFIG_PPC_BOOK3S_32 */