mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-12-16 15:34:48 +08:00
fb799447ae
Mark reported that the ORC unwinder incorrectly marks an unwind as
reliable when the unwind terminates prematurely in the dark corners of
return_to_handler() due to lack of information about the next frame.
The problem is UNWIND_HINT_EMPTY is used in two different situations:
1) The end of the kernel stack unwind before hitting user entry, boot
code, or fork entry
2) A blind spot in ORC coverage where the unwinder has to bail due to
lack of information about the next frame
The ORC unwinder has no way to tell the difference between the two.
When it encounters an undefined stack state with 'end=1', it blindly
marks the stack reliable, which can break the livepatch consistency
model.
Fix it by splitting UNWIND_HINT_EMPTY into UNWIND_HINT_UNDEFINED and
UNWIND_HINT_END_OF_STACK.
Reported-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/fd6212c8b450d3564b855e1cb48404d6277b4d9f.1677683419.git.jpoimboe@kernel.org
310 lines
7.5 KiB
ArmAsm
310 lines
7.5 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* Asm versions of Xen pv-ops, suitable for direct use.
|
|
*
|
|
* We only bother with direct forms (ie, vcpu in percpu data) of the
|
|
* operations here; the indirect forms are better handled in C.
|
|
*/
|
|
|
|
#include <asm/errno.h>
|
|
#include <asm/asm-offsets.h>
|
|
#include <asm/percpu.h>
|
|
#include <asm/processor-flags.h>
|
|
#include <asm/segment.h>
|
|
#include <asm/thread_info.h>
|
|
#include <asm/asm.h>
|
|
#include <asm/frame.h>
|
|
#include <asm/unwind_hints.h>
|
|
|
|
#include <xen/interface/xen.h>
|
|
|
|
#include <linux/init.h>
|
|
#include <linux/linkage.h>
|
|
#include <../entry/calling.h>
|
|
|
|
.pushsection .noinstr.text, "ax"
|
|
/*
|
|
* Disabling events is simply a matter of making the event mask
|
|
* non-zero.
|
|
*/
|
|
SYM_FUNC_START(xen_irq_disable_direct)
|
|
movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
|
RET
|
|
SYM_FUNC_END(xen_irq_disable_direct)
|
|
|
|
/*
|
|
* Force an event check by making a hypercall, but preserve regs
|
|
* before making the call.
|
|
*/
|
|
SYM_FUNC_START(check_events)
|
|
FRAME_BEGIN
|
|
push %rax
|
|
push %rcx
|
|
push %rdx
|
|
push %rsi
|
|
push %rdi
|
|
push %r8
|
|
push %r9
|
|
push %r10
|
|
push %r11
|
|
call xen_force_evtchn_callback
|
|
pop %r11
|
|
pop %r10
|
|
pop %r9
|
|
pop %r8
|
|
pop %rdi
|
|
pop %rsi
|
|
pop %rdx
|
|
pop %rcx
|
|
pop %rax
|
|
FRAME_END
|
|
RET
|
|
SYM_FUNC_END(check_events)
|
|
|
|
/*
|
|
* Enable events. This clears the event mask and tests the pending
|
|
* event status with one and operation. If there are pending events,
|
|
* then enter the hypervisor to get them handled.
|
|
*/
|
|
SYM_FUNC_START(xen_irq_enable_direct)
|
|
FRAME_BEGIN
|
|
/* Unmask events */
|
|
movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
|
|
|
/*
|
|
* Preempt here doesn't matter because that will deal with any
|
|
* pending interrupts. The pending check may end up being run
|
|
* on the wrong CPU, but that doesn't hurt.
|
|
*/
|
|
|
|
/* Test for pending */
|
|
testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
|
|
jz 1f
|
|
|
|
call check_events
|
|
1:
|
|
FRAME_END
|
|
RET
|
|
SYM_FUNC_END(xen_irq_enable_direct)
|
|
|
|
/*
|
|
* (xen_)save_fl is used to get the current interrupt enable status.
|
|
* Callers expect the status to be in X86_EFLAGS_IF, and other bits
|
|
* may be set in the return value. We take advantage of this by
|
|
* making sure that X86_EFLAGS_IF has the right value (and other bits
|
|
* in that byte are 0), but other bits in the return value are
|
|
* undefined. We need to toggle the state of the bit, because Xen and
|
|
* x86 use opposite senses (mask vs enable).
|
|
*/
|
|
SYM_FUNC_START(xen_save_fl_direct)
|
|
testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
|
setz %ah
|
|
addb %ah, %ah
|
|
RET
|
|
SYM_FUNC_END(xen_save_fl_direct)
|
|
|
|
SYM_FUNC_START(xen_read_cr2)
|
|
FRAME_BEGIN
|
|
_ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
|
|
_ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
|
|
FRAME_END
|
|
RET
|
|
SYM_FUNC_END(xen_read_cr2);
|
|
|
|
SYM_FUNC_START(xen_read_cr2_direct)
|
|
FRAME_BEGIN
|
|
_ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX
|
|
FRAME_END
|
|
RET
|
|
SYM_FUNC_END(xen_read_cr2_direct);
|
|
.popsection
|
|
|
|
.macro xen_pv_trap name
|
|
SYM_CODE_START(xen_\name)
|
|
UNWIND_HINT_ENTRY
|
|
ENDBR
|
|
pop %rcx
|
|
pop %r11
|
|
jmp \name
|
|
SYM_CODE_END(xen_\name)
|
|
_ASM_NOKPROBE(xen_\name)
|
|
.endm
|
|
|
|
xen_pv_trap asm_exc_divide_error
|
|
xen_pv_trap asm_xenpv_exc_debug
|
|
xen_pv_trap asm_exc_int3
|
|
xen_pv_trap asm_xenpv_exc_nmi
|
|
xen_pv_trap asm_exc_overflow
|
|
xen_pv_trap asm_exc_bounds
|
|
xen_pv_trap asm_exc_invalid_op
|
|
xen_pv_trap asm_exc_device_not_available
|
|
xen_pv_trap asm_xenpv_exc_double_fault
|
|
xen_pv_trap asm_exc_coproc_segment_overrun
|
|
xen_pv_trap asm_exc_invalid_tss
|
|
xen_pv_trap asm_exc_segment_not_present
|
|
xen_pv_trap asm_exc_stack_segment
|
|
xen_pv_trap asm_exc_general_protection
|
|
xen_pv_trap asm_exc_page_fault
|
|
xen_pv_trap asm_exc_spurious_interrupt_bug
|
|
xen_pv_trap asm_exc_coprocessor_error
|
|
xen_pv_trap asm_exc_alignment_check
|
|
#ifdef CONFIG_X86_KERNEL_IBT
|
|
xen_pv_trap asm_exc_control_protection
|
|
#endif
|
|
#ifdef CONFIG_X86_MCE
|
|
xen_pv_trap asm_xenpv_exc_machine_check
|
|
#endif /* CONFIG_X86_MCE */
|
|
xen_pv_trap asm_exc_simd_coprocessor_error
|
|
#ifdef CONFIG_IA32_EMULATION
|
|
xen_pv_trap entry_INT80_compat
|
|
#endif
|
|
xen_pv_trap asm_exc_xen_unknown_trap
|
|
xen_pv_trap asm_exc_xen_hypervisor_callback
|
|
|
|
__INIT
|
|
SYM_CODE_START(xen_early_idt_handler_array)
|
|
i = 0
|
|
.rept NUM_EXCEPTION_VECTORS
|
|
UNWIND_HINT_UNDEFINED
|
|
ENDBR
|
|
pop %rcx
|
|
pop %r11
|
|
jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
|
|
i = i + 1
|
|
.fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
|
|
.endr
|
|
SYM_CODE_END(xen_early_idt_handler_array)
|
|
__FINIT
|
|
|
|
hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
|
|
/*
|
|
* Xen64 iret frame:
|
|
*
|
|
* ss
|
|
* rsp
|
|
* rflags
|
|
* cs
|
|
* rip <-- standard iret frame
|
|
*
|
|
* flags
|
|
*
|
|
* rcx }
|
|
* r11 }<-- pushed by hypercall page
|
|
* rsp->rax }
|
|
*/
|
|
SYM_CODE_START(xen_iret)
|
|
UNWIND_HINT_UNDEFINED
|
|
ANNOTATE_NOENDBR
|
|
pushq $0
|
|
jmp hypercall_iret
|
|
SYM_CODE_END(xen_iret)
|
|
|
|
/*
|
|
* XEN pv doesn't use trampoline stack, PER_CPU_VAR(cpu_tss_rw + TSS_sp0) is
|
|
* also the kernel stack. Reusing swapgs_restore_regs_and_return_to_usermode()
|
|
* in XEN pv would cause %rsp to move up to the top of the kernel stack and
|
|
* leave the IRET frame below %rsp, which is dangerous to be corrupted if #NMI
|
|
* interrupts. And swapgs_restore_regs_and_return_to_usermode() pushing the IRET
|
|
* frame at the same address is useless.
|
|
*/
|
|
SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode)
|
|
UNWIND_HINT_REGS
|
|
POP_REGS
|
|
|
|
/* stackleak_erase() can work safely on the kernel stack. */
|
|
STACKLEAK_ERASE_NOCLOBBER
|
|
|
|
addq $8, %rsp /* skip regs->orig_ax */
|
|
jmp xen_iret
|
|
SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)
|
|
|
|
/*
|
|
* Xen handles syscall callbacks much like ordinary exceptions, which
|
|
* means we have:
|
|
* - kernel gs
|
|
* - kernel rsp
|
|
* - an iret-like stack frame on the stack (including rcx and r11):
|
|
* ss
|
|
* rsp
|
|
* rflags
|
|
* cs
|
|
* rip
|
|
* r11
|
|
* rsp->rcx
|
|
*/
|
|
|
|
/* Normal 64-bit system call target */
|
|
SYM_CODE_START(xen_entry_SYSCALL_64)
|
|
UNWIND_HINT_ENTRY
|
|
ENDBR
|
|
popq %rcx
|
|
popq %r11
|
|
|
|
/*
|
|
* Neither Xen nor the kernel really knows what the old SS and
|
|
* CS were. The kernel expects __USER_DS and __USER_CS, so
|
|
* report those values even though Xen will guess its own values.
|
|
*/
|
|
movq $__USER_DS, 4*8(%rsp)
|
|
movq $__USER_CS, 1*8(%rsp)
|
|
|
|
jmp entry_SYSCALL_64_after_hwframe
|
|
SYM_CODE_END(xen_entry_SYSCALL_64)
|
|
|
|
#ifdef CONFIG_IA32_EMULATION
|
|
|
|
/* 32-bit compat syscall target */
|
|
SYM_CODE_START(xen_entry_SYSCALL_compat)
|
|
UNWIND_HINT_ENTRY
|
|
ENDBR
|
|
popq %rcx
|
|
popq %r11
|
|
|
|
/*
|
|
* Neither Xen nor the kernel really knows what the old SS and
|
|
* CS were. The kernel expects __USER_DS and __USER32_CS, so
|
|
* report those values even though Xen will guess its own values.
|
|
*/
|
|
movq $__USER_DS, 4*8(%rsp)
|
|
movq $__USER32_CS, 1*8(%rsp)
|
|
|
|
jmp entry_SYSCALL_compat_after_hwframe
|
|
SYM_CODE_END(xen_entry_SYSCALL_compat)
|
|
|
|
/* 32-bit compat sysenter target */
|
|
SYM_CODE_START(xen_entry_SYSENTER_compat)
|
|
UNWIND_HINT_ENTRY
|
|
ENDBR
|
|
/*
|
|
* NB: Xen is polite and clears TF from EFLAGS for us. This means
|
|
* that we don't need to guard against single step exceptions here.
|
|
*/
|
|
popq %rcx
|
|
popq %r11
|
|
|
|
/*
|
|
* Neither Xen nor the kernel really knows what the old SS and
|
|
* CS were. The kernel expects __USER_DS and __USER32_CS, so
|
|
* report those values even though Xen will guess its own values.
|
|
*/
|
|
movq $__USER_DS, 4*8(%rsp)
|
|
movq $__USER32_CS, 1*8(%rsp)
|
|
|
|
jmp entry_SYSENTER_compat_after_hwframe
|
|
SYM_CODE_END(xen_entry_SYSENTER_compat)
|
|
|
|
#else /* !CONFIG_IA32_EMULATION */
|
|
|
|
SYM_CODE_START(xen_entry_SYSCALL_compat)
|
|
SYM_CODE_START(xen_entry_SYSENTER_compat)
|
|
UNWIND_HINT_ENTRY
|
|
ENDBR
|
|
lea 16(%rsp), %rsp /* strip %rcx, %r11 */
|
|
mov $-ENOSYS, %rax
|
|
pushq $0
|
|
jmp hypercall_iret
|
|
SYM_CODE_END(xen_entry_SYSENTER_compat)
|
|
SYM_CODE_END(xen_entry_SYSCALL_compat)
|
|
|
|
#endif /* CONFIG_IA32_EMULATION */
|