2008-07-09 06:06:46 +08:00
|
|
|
/*
|
2009-02-05 23:57:48 +08:00
|
|
|
* Asm versions of Xen pv-ops, suitable for either direct use or
|
|
|
|
* inlining. The inline versions are the same as the direct-use
|
|
|
|
* versions, with the pre- and post-amble chopped off.
|
|
|
|
*
|
|
|
|
* This code is encoded for size rather than absolute efficiency, with
|
|
|
|
* a view to being able to inline as much as possible.
|
|
|
|
*
|
|
|
|
* We only bother with direct forms (ie, vcpu in pda) of the
|
|
|
|
* operations here; the indirect forms are better handled in C, since
|
|
|
|
* they're generally too large to inline anyway.
|
2008-07-09 06:06:46 +08:00
|
|
|
*/
|
|
|
|
|
2008-07-09 06:07:14 +08:00
|
|
|
#include <asm/errno.h>
|
2009-01-18 23:38:58 +08:00
|
|
|
#include <asm/percpu.h>
|
2009-02-03 05:55:42 +08:00
|
|
|
#include <asm/processor-flags.h>
|
|
|
|
#include <asm/segment.h>
|
2015-04-24 23:31:33 +08:00
|
|
|
#include <asm/asm-offsets.h>
|
2015-04-24 23:31:35 +08:00
|
|
|
#include <asm/thread_info.h>
|
2008-07-09 06:06:46 +08:00
|
|
|
|
|
|
|
#include <xen/interface/xen.h>
|
|
|
|
|
2009-02-03 05:55:42 +08:00
|
|
|
#include "xen-asm.h"
|
2008-07-09 06:06:46 +08:00
|
|
|
|
2008-07-09 06:07:00 +08:00
|
|
|
ENTRY(xen_adjust_exception_frame)
|
2009-02-05 23:57:48 +08:00
|
|
|
mov 8+0(%rsp), %rcx
|
|
|
|
mov 8+8(%rsp), %r11
|
2008-07-09 06:07:00 +08:00
|
|
|
ret $16
|
|
|
|
|
2008-07-09 06:07:14 +08:00
|
|
|
hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
|
|
|
|
/*
|
2009-02-05 23:57:48 +08:00
|
|
|
* Xen64 iret frame:
|
|
|
|
*
|
|
|
|
* ss
|
|
|
|
* rsp
|
|
|
|
* rflags
|
|
|
|
* cs
|
|
|
|
* rip <-- standard iret frame
|
|
|
|
*
|
|
|
|
* flags
|
|
|
|
*
|
|
|
|
* rcx }
|
|
|
|
* r11 }<-- pushed by hypercall page
|
|
|
|
* rsp->rax }
|
2008-07-09 06:07:14 +08:00
|
|
|
*/
|
2008-07-09 06:06:46 +08:00
|
|
|
ENTRY(xen_iret)
|
|
|
|
pushq $0
|
2008-07-09 06:07:14 +08:00
|
|
|
1: jmp hypercall_iret
|
|
|
|
ENDPATCH(xen_iret)
|
|
|
|
RELOC(xen_iret, 1b+1)
|
2008-07-09 06:06:46 +08:00
|
|
|
|
2008-07-09 06:07:14 +08:00
|
|
|
ENTRY(xen_sysret64)
|
2009-02-05 23:57:48 +08:00
|
|
|
/*
|
|
|
|
* We're already on the usermode stack at this point, but
|
|
|
|
* still with the kernel gs, so we can easily switch back
|
|
|
|
*/
|
2015-03-17 21:42:59 +08:00
|
|
|
movq %rsp, PER_CPU_VAR(rsp_scratch)
|
2015-04-24 23:31:35 +08:00
|
|
|
movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
|
2008-07-09 06:07:14 +08:00
|
|
|
|
|
|
|
pushq $__USER_DS
|
2015-03-17 21:42:59 +08:00
|
|
|
pushq PER_CPU_VAR(rsp_scratch)
|
2008-07-09 06:07:14 +08:00
|
|
|
pushq %r11
|
|
|
|
pushq $__USER_CS
|
|
|
|
pushq %rcx
|
|
|
|
|
|
|
|
pushq $VGCF_in_syscall
|
|
|
|
1: jmp hypercall_iret
|
|
|
|
ENDPATCH(xen_sysret64)
|
|
|
|
RELOC(xen_sysret64, 1b+1)
|
|
|
|
|
|
|
|
ENTRY(xen_sysret32)
|
2009-02-05 23:57:48 +08:00
|
|
|
/*
|
|
|
|
* We're already on the usermode stack at this point, but
|
|
|
|
* still with the kernel gs, so we can easily switch back
|
|
|
|
*/
|
2015-03-17 21:42:59 +08:00
|
|
|
movq %rsp, PER_CPU_VAR(rsp_scratch)
|
2015-04-24 23:31:35 +08:00
|
|
|
movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
|
2008-07-09 06:07:14 +08:00
|
|
|
|
|
|
|
pushq $__USER32_DS
|
2015-03-17 21:42:59 +08:00
|
|
|
pushq PER_CPU_VAR(rsp_scratch)
|
2008-07-09 06:07:14 +08:00
|
|
|
pushq %r11
|
|
|
|
pushq $__USER32_CS
|
|
|
|
pushq %rcx
|
|
|
|
|
2009-11-26 05:15:38 +08:00
|
|
|
pushq $0
|
2008-07-09 06:07:14 +08:00
|
|
|
1: jmp hypercall_iret
|
|
|
|
ENDPATCH(xen_sysret32)
|
|
|
|
RELOC(xen_sysret32, 1b+1)
|
|
|
|
|
|
|
|
/*
|
2009-02-05 23:57:48 +08:00
|
|
|
* Xen handles syscall callbacks much like ordinary exceptions, which
|
|
|
|
* means we have:
|
|
|
|
* - kernel gs
|
|
|
|
* - kernel rsp
|
|
|
|
* - an iret-like stack frame on the stack (including rcx and r11):
|
|
|
|
* ss
|
|
|
|
* rsp
|
|
|
|
* rflags
|
|
|
|
* cs
|
|
|
|
* rip
|
|
|
|
* r11
|
|
|
|
* rsp->rcx
|
|
|
|
*
|
|
|
|
* In all the entrypoints, we undo all that to make it look like a
|
|
|
|
* CPU-generated syscall/sysenter and jump to the normal entrypoint.
|
2008-07-09 06:07:14 +08:00
|
|
|
*/
|
|
|
|
|
|
|
|
.macro undo_xen_syscall
|
2009-02-05 23:57:48 +08:00
|
|
|
mov 0*8(%rsp), %rcx
|
|
|
|
mov 1*8(%rsp), %r11
|
|
|
|
mov 5*8(%rsp), %rsp
|
2008-07-09 06:07:14 +08:00
|
|
|
.endm
|
|
|
|
|
|
|
|
/* Normal 64-bit system call target */
|
|
|
|
ENTRY(xen_syscall_target)
|
|
|
|
undo_xen_syscall
|
|
|
|
jmp system_call_after_swapgs
|
|
|
|
ENDPROC(xen_syscall_target)
|
|
|
|
|
|
|
|
#ifdef CONFIG_IA32_EMULATION
|
|
|
|
|
|
|
|
/* 32-bit compat syscall target */
|
|
|
|
ENTRY(xen_syscall32_target)
|
|
|
|
undo_xen_syscall
|
2015-06-08 14:28:07 +08:00
|
|
|
jmp entry_SYSCALL_compat
|
2008-07-09 06:07:14 +08:00
|
|
|
ENDPROC(xen_syscall32_target)
|
|
|
|
|
|
|
|
/* 32-bit compat sysenter target */
|
|
|
|
ENTRY(xen_sysenter_target)
|
|
|
|
undo_xen_syscall
|
2015-06-08 14:33:56 +08:00
|
|
|
jmp entry_SYSENTER_compat
|
2008-07-09 06:07:14 +08:00
|
|
|
ENDPROC(xen_sysenter_target)
|
|
|
|
|
|
|
|
#else /* !CONFIG_IA32_EMULATION */
|
|
|
|
|
|
|
|
ENTRY(xen_syscall32_target)
|
|
|
|
ENTRY(xen_sysenter_target)
|
2009-02-05 23:57:48 +08:00
|
|
|
lea 16(%rsp), %rsp /* strip %rcx, %r11 */
|
2008-07-09 06:07:14 +08:00
|
|
|
mov $-ENOSYS, %rax
|
2009-11-26 05:15:38 +08:00
|
|
|
pushq $0
|
2008-07-09 06:07:14 +08:00
|
|
|
jmp hypercall_iret
|
|
|
|
ENDPROC(xen_syscall32_target)
|
|
|
|
ENDPROC(xen_sysenter_target)
|
|
|
|
|
|
|
|
#endif /* CONFIG_IA32_EMULATION */
|