xen-asm.S

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Asm versions of Xen pv-ops, suitable for direct use.
 *
 * We only bother with direct forms (ie, vcpu in percpu data) of the
 * operations here; the indirect forms are better handled in C.
 */

#include <asm/errno.h>
#include <asm/asm-offsets.h>
#include <asm/percpu.h>
#include <asm/processor-flags.h>
#include <asm/segment.h>
#include <asm/thread_info.h>
#include <asm/asm.h>
#include <asm/frame.h>
#include <asm/unwind_hints.h>

#include <xen/interface/xen.h>

#include <linux/init.h>
#include <linux/linkage.h>
#include <linux/objtool.h>
#include <../entry/calling.h>

.pushsection .noinstr.text, "ax"
/*
 * PV hypercall interface to the hypervisor.
 *
 * Called via inline asm(), so better preserve %rcx and %r11.
 *
 * Input:
 *	%eax: hypercall number
 *	%rdi, %rsi, %rdx, %r10, %r8: args 1..5 for the hypercall
 * Output: %rax
 */
SYM_FUNC_START(xen_hypercall_pv)
	ANNOTATE_NOENDBR
	push %rcx
	push %r11
	UNWIND_HINT_SAVE
	syscall
	UNWIND_HINT_RESTORE
	pop %r11
	pop %rcx
	RET
SYM_FUNC_END(xen_hypercall_pv)

/*
 * Disabling events is simply a matter of making the event mask
 * non-zero.
 */
SYM_FUNC_START(xen_irq_disable_direct)
	ENDBR
	movb $1, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
	RET
SYM_FUNC_END(xen_irq_disable_direct)

/*
 * Force an event check by making a hypercall, but preserve regs
 * before making the call.
 */
SYM_FUNC_START(check_events)
	FRAME_BEGIN
	push %rax
	push %rcx
	push %rdx
	push %rsi
	push %rdi
	push %r8
	push %r9
	push %r10
	push %r11
	call xen_force_evtchn_callback
	pop %r11
	pop %r10
	pop %r9
	pop %r8
	pop %rdi
	pop %rsi
	pop %rdx
	pop %rcx
	pop %rax
	FRAME_END
	RET
SYM_FUNC_END(check_events)

/*
 * Enable events.  This clears the event mask and tests the pending
 * event status with one and operation.  If there are pending events,
 * then enter the hypervisor to get them handled.
 */
SYM_FUNC_START(xen_irq_enable_direct)
	ENDBR
	FRAME_BEGIN
	/* Unmask events */
	movb $0, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)

	/*
	 * Preempt here doesn't matter because that will deal with any
	 * pending interrupts.  The pending check may end up being run
	 * on the wrong CPU, but that doesn't hurt.
	 */

	/* Test for pending */
	testb $0xff, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_pending)
	jz 1f

	call check_events
1:
	FRAME_END
	RET
SYM_FUNC_END(xen_irq_enable_direct)

/*
 * (xen_)save_fl is used to get the current interrupt enable status.
 * Callers expect the status to be in X86_EFLAGS_IF, and other bits
 * may be set in the return value.  We take advantage of this by
 * making sure that X86_EFLAGS_IF has the right value (and other bits
 * in that byte are 0), but other bits in the return value are
 * undefined.  We need to toggle the state of the bit, because Xen and
 * x86 use opposite senses (mask vs enable).
 */
SYM_FUNC_START(xen_save_fl_direct)
	ENDBR
	testb $0xff, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
	setz %ah
	addb %ah, %ah
	RET
SYM_FUNC_END(xen_save_fl_direct)

SYM_FUNC_START(xen_read_cr2)
	ENDBR
	FRAME_BEGIN
	_ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
	_ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
	FRAME_END
	RET
SYM_FUNC_END(xen_read_cr2);

SYM_FUNC_START(xen_read_cr2_direct)
	ENDBR
	FRAME_BEGIN
	_ASM_MOV PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_arch_cr2), %_ASM_AX
	FRAME_END
	RET
SYM_FUNC_END(xen_read_cr2_direct);
.popsection

.macro xen_pv_trap name
SYM_CODE_START(xen_\name)
	UNWIND_HINT_ENTRY
	ENDBR
	pop %rcx
	pop %r11
	jmp  \name
SYM_CODE_END(xen_\name)
_ASM_NOKPROBE(xen_\name)
.endm

xen_pv_trap asm_exc_divide_error
xen_pv_trap asm_xenpv_exc_debug
xen_pv_trap asm_exc_int3
xen_pv_trap asm_xenpv_exc_nmi
xen_pv_trap asm_exc_overflow
xen_pv_trap asm_exc_bounds
xen_pv_trap asm_exc_invalid_op
xen_pv_trap asm_exc_device_not_available
xen_pv_trap asm_xenpv_exc_double_fault
xen_pv_trap asm_exc_coproc_segment_overrun
xen_pv_trap asm_exc_invalid_tss
xen_pv_trap asm_exc_segment_not_present
xen_pv_trap asm_exc_stack_segment
xen_pv_trap asm_exc_general_protection
xen_pv_trap asm_exc_page_fault
xen_pv_trap asm_exc_spurious_interrupt_bug
xen_pv_trap asm_exc_coprocessor_error
xen_pv_trap asm_exc_alignment_check
#ifdef CONFIG_X86_CET
xen_pv_trap asm_exc_control_protection
#endif
#ifdef CONFIG_X86_MCE
xen_pv_trap asm_xenpv_exc_machine_check
#endif /* CONFIG_X86_MCE */
xen_pv_trap asm_exc_simd_coprocessor_error
#ifdef CONFIG_IA32_EMULATION
xen_pv_trap asm_int80_emulation
#endif
xen_pv_trap asm_exc_xen_unknown_trap
xen_pv_trap asm_exc_xen_hypervisor_callback

	__INIT
SYM_CODE_START(xen_early_idt_handler_array)
	i = 0
	.rept NUM_EXCEPTION_VECTORS
	UNWIND_HINT_UNDEFINED
	ENDBR
	pop %rcx
	pop %r11
	jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
	i = i + 1
	.fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
	.endr
SYM_CODE_END(xen_early_idt_handler_array)
	__FINIT

/*
 * Xen64 iret frame:
 *
 *	ss
 *	rsp
 *	rflags
 *	cs
 *	rip		<-- standard iret frame
 *
 *	flags		<-- xen_iret must push from here on
 *
 *	rcx
 *	r11
 * rsp->rax
 */
.macro xen_hypercall_iret
	pushq $0	/* Flags */
	push %rcx
	push %r11
	push %rax
	mov  $__HYPERVISOR_iret, %eax
	syscall		/* Do the IRET. */
	ud2		/* The SYSCALL should never return. */
.endm

SYM_CODE_START(xen_iret)
	UNWIND_HINT_UNDEFINED
	ANNOTATE_NOENDBR
	xen_hypercall_iret
SYM_CODE_END(xen_iret)

/*
 * XEN pv doesn't use trampoline stack, PER_CPU_VAR(cpu_tss_rw + TSS_sp0) is
 * also the kernel stack.  Reusing swapgs_restore_regs_and_return_to_usermode()
 * in XEN pv would cause %rsp to move up to the top of the kernel stack and
 * leave the IRET frame below %rsp, which is dangerous to be corrupted if #NMI
 * interrupts. And swapgs_restore_regs_and_return_to_usermode() pushing the IRET
 * frame at the same address is useless.
 */
SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode)
	UNWIND_HINT_REGS
	POP_REGS

	/* stackleak_erase() can work safely on the kernel stack. */
	STACKLEAK_ERASE_NOCLOBBER

	addq	$8, %rsp	/* skip regs->orig_ax */
	jmp xen_iret
SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)

/*
 * Xen handles syscall callbacks much like ordinary exceptions, which
 * means we have:
 * - kernel gs
 * - kernel rsp
 * - an iret-like stack frame on the stack (including rcx and r11):
 *	ss
 *	rsp
 *	rflags
 *	cs
 *	rip
 *	r11
 * rsp->rcx
 */

/* Normal 64-bit system call target */
SYM_CODE_START(xen_entry_SYSCALL_64)
	UNWIND_HINT_ENTRY
	ENDBR
	popq %rcx
	popq %r11

	/*
	 * Neither Xen nor the kernel really knows what the old SS and
	 * CS were.  The kernel expects __USER_DS and __USER_CS, so
	 * report those values even though Xen will guess its own values.
	 */
	movq $__USER_DS, 4*8(%rsp)
	movq $__USER_CS, 1*8(%rsp)

	jmp entry_SYSCALL_64_after_hwframe
SYM_CODE_END(xen_entry_SYSCALL_64)

#ifdef CONFIG_IA32_EMULATION

/* 32-bit compat syscall target */
SYM_CODE_START(xen_entry_SYSCALL_compat)
	UNWIND_HINT_ENTRY
	ENDBR
	popq %rcx
	popq %r11

	/*
	 * Neither Xen nor the kernel really knows what the old SS and
	 * CS were.  The kernel expects __USER_DS and __USER32_CS, so
	 * report those values even though Xen will guess its own values.
	 */
	movq $__USER_DS, 4*8(%rsp)
	movq $__USER32_CS, 1*8(%rsp)

	jmp entry_SYSCALL_compat_after_hwframe
SYM_CODE_END(xen_entry_SYSCALL_compat)

/* 32-bit compat sysenter target */
SYM_CODE_START(xen_entry_SYSENTER_compat)
	UNWIND_HINT_ENTRY
	ENDBR
	/*
	 * NB: Xen is polite and clears TF from EFLAGS for us.  This means
	 * that we don't need to guard against single step exceptions here.
	 */
	popq %rcx
	popq %r11

	/*
	 * Neither Xen nor the kernel really knows what the old SS and
	 * CS were.  The kernel expects __USER_DS and __USER32_CS, so
	 * report those values even though Xen will guess its own values.
	 */
	movq $__USER_DS, 4*8(%rsp)
	movq $__USER32_CS, 1*8(%rsp)

	jmp entry_SYSENTER_compat_after_hwframe
SYM_CODE_END(xen_entry_SYSENTER_compat)

#else /* !CONFIG_IA32_EMULATION */

SYM_CODE_START(xen_entry_SYSCALL_compat)
SYM_CODE_START(xen_entry_SYSENTER_compat)
	UNWIND_HINT_ENTRY
	ENDBR
	lea 16(%rsp), %rsp	/* strip %rcx, %r11 */
	mov $-ENOSYS, %rax
	xen_hypercall_iret
SYM_CODE_END(xen_entry_SYSENTER_compat)
SYM_CODE_END(xen_entry_SYSCALL_compat)

#endif	/* CONFIG_IA32_EMULATION */